You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

658 lines
26 KiB

4 years ago
  1. # Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License"). You
  4. # may not use this file except in compliance with the License. A copy of
  5. # the License is located at
  6. #
  7. # http://aws.amazon.com/apache2.0/
  8. #
  9. # or in the "license" file accompanying this file. This file is
  10. # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
  11. # ANY KIND, either express or implied. See the License for the specific
  12. # language governing permissions and limitations under the License.
  13. import copy
  14. import logging
  15. import threading
  16. from botocore.compat import six
  17. from s3transfer.utils import get_callbacks
  18. from s3transfer.utils import signal_transferring
  19. from s3transfer.utils import signal_not_transferring
  20. from s3transfer.utils import CallArgs
  21. from s3transfer.utils import OSUtils
  22. from s3transfer.utils import TaskSemaphore
  23. from s3transfer.utils import SlidingWindowSemaphore
  24. from s3transfer.exceptions import CancelledError
  25. from s3transfer.exceptions import FatalError
  26. from s3transfer.futures import IN_MEMORY_DOWNLOAD_TAG
  27. from s3transfer.futures import IN_MEMORY_UPLOAD_TAG
  28. from s3transfer.futures import BoundedExecutor
  29. from s3transfer.futures import TransferFuture
  30. from s3transfer.futures import TransferMeta
  31. from s3transfer.futures import TransferCoordinator
  32. from s3transfer.download import DownloadSubmissionTask
  33. from s3transfer.upload import UploadSubmissionTask
  34. from s3transfer.copies import CopySubmissionTask
  35. from s3transfer.delete import DeleteSubmissionTask
  36. from s3transfer.bandwidth import LeakyBucket
  37. from s3transfer.bandwidth import BandwidthLimiter
  38. KB = 1024
  39. MB = KB * KB
  40. logger = logging.getLogger(__name__)
  41. class TransferConfig(object):
  42. def __init__(self,
  43. multipart_threshold=8 * MB,
  44. multipart_chunksize=8 * MB,
  45. max_request_concurrency=10,
  46. max_submission_concurrency=5,
  47. max_request_queue_size=1000,
  48. max_submission_queue_size=1000,
  49. max_io_queue_size=1000,
  50. io_chunksize=256 * KB,
  51. num_download_attempts=5,
  52. max_in_memory_upload_chunks=10,
  53. max_in_memory_download_chunks=10,
  54. max_bandwidth=None):
  55. """Configurations for the transfer mangager
  56. :param multipart_threshold: The threshold for which multipart
  57. transfers occur.
  58. :param max_request_concurrency: The maximum number of S3 API
  59. transfer-related requests that can happen at a time.
  60. :param max_submission_concurrency: The maximum number of threads
  61. processing a call to a TransferManager method. Processing a
  62. call usually entails determining which S3 API requests that need
  63. to be enqueued, but does **not** entail making any of the
  64. S3 API data transfering requests needed to perform the transfer.
  65. The threads controlled by ``max_request_concurrency`` is
  66. responsible for that.
  67. :param multipart_chunksize: The size of each transfer if a request
  68. becomes a multipart transfer.
  69. :param max_request_queue_size: The maximum amount of S3 API requests
  70. that can be queued at a time. A value of zero means that there
  71. is no maximum.
  72. :param max_submission_queue_size: The maximum amount of
  73. TransferManager method calls that can be queued at a time. A value
  74. of zero means that there is no maximum.
  75. :param max_io_queue_size: The maximum amount of read parts that
  76. can be queued to be written to disk per download. A value of zero
  77. means that there is no maximum. The default size for each element
  78. in this queue is 8 KB.
  79. :param io_chunksize: The max size of each chunk in the io queue.
  80. Currently, this is size used when reading from the downloaded
  81. stream as well.
  82. :param num_download_attempts: The number of download attempts that
  83. will be tried upon errors with downloading an object in S3. Note
  84. that these retries account for errors that occur when streamming
  85. down the data from s3 (i.e. socket errors and read timeouts that
  86. occur after recieving an OK response from s3).
  87. Other retryable exceptions such as throttling errors and 5xx errors
  88. are already retried by botocore (this default is 5). The
  89. ``num_download_attempts`` does not take into account the
  90. number of exceptions retried by botocore.
  91. :param max_in_memory_upload_chunks: The number of chunks that can
  92. be stored in memory at a time for all ongoing upload requests.
  93. This pertains to chunks of data that need to be stored in memory
  94. during an upload if the data is sourced from a file-like object.
  95. The total maximum memory footprint due to a in-memory upload
  96. chunks is roughly equal to:
  97. max_in_memory_upload_chunks * multipart_chunksize
  98. + max_submission_concurrency * multipart_chunksize
  99. ``max_submission_concurrency`` has an affect on this value because
  100. for each thread pulling data off of a file-like object, they may
  101. be waiting with a single read chunk to be submitted for upload
  102. because the ``max_in_memory_upload_chunks`` value has been reached
  103. by the threads making the upload request.
  104. :param max_in_memory_download_chunks: The number of chunks that can
  105. be buffered in memory and **not** in the io queue at a time for all
  106. ongoing dowload requests. This pertains specifically to file-like
  107. objects that cannot be seeked. The total maximum memory footprint
  108. due to a in-memory download chunks is roughly equal to:
  109. max_in_memory_download_chunks * multipart_chunksize
  110. :param max_bandwidth: The maximum bandwidth that will be consumed
  111. in uploading and downloading file content. The value is in terms of
  112. bytes per second.
  113. """
  114. self.multipart_threshold = multipart_threshold
  115. self.multipart_chunksize = multipart_chunksize
  116. self.max_request_concurrency = max_request_concurrency
  117. self.max_submission_concurrency = max_submission_concurrency
  118. self.max_request_queue_size = max_request_queue_size
  119. self.max_submission_queue_size = max_submission_queue_size
  120. self.max_io_queue_size = max_io_queue_size
  121. self.io_chunksize = io_chunksize
  122. self.num_download_attempts = num_download_attempts
  123. self.max_in_memory_upload_chunks = max_in_memory_upload_chunks
  124. self.max_in_memory_download_chunks = max_in_memory_download_chunks
  125. self.max_bandwidth = max_bandwidth
  126. self._validate_attrs_are_nonzero()
  127. def _validate_attrs_are_nonzero(self):
  128. for attr, attr_val, in self.__dict__.items():
  129. if attr_val is not None and attr_val <= 0:
  130. raise ValueError(
  131. 'Provided parameter %s of value %s must be greater than '
  132. '0.' % (attr, attr_val))
  133. class TransferManager(object):
  134. ALLOWED_DOWNLOAD_ARGS = [
  135. 'VersionId',
  136. 'SSECustomerAlgorithm',
  137. 'SSECustomerKey',
  138. 'SSECustomerKeyMD5',
  139. 'RequestPayer',
  140. ]
  141. ALLOWED_UPLOAD_ARGS = [
  142. 'ACL',
  143. 'CacheControl',
  144. 'ContentDisposition',
  145. 'ContentEncoding',
  146. 'ContentLanguage',
  147. 'ContentType',
  148. 'Expires',
  149. 'GrantFullControl',
  150. 'GrantRead',
  151. 'GrantReadACP',
  152. 'GrantWriteACP',
  153. 'Metadata',
  154. 'RequestPayer',
  155. 'ServerSideEncryption',
  156. 'StorageClass',
  157. 'SSECustomerAlgorithm',
  158. 'SSECustomerKey',
  159. 'SSECustomerKeyMD5',
  160. 'SSEKMSKeyId',
  161. 'WebsiteRedirectLocation'
  162. ]
  163. ALLOWED_COPY_ARGS = ALLOWED_UPLOAD_ARGS + [
  164. 'CopySourceIfMatch',
  165. 'CopySourceIfModifiedSince',
  166. 'CopySourceIfNoneMatch',
  167. 'CopySourceIfUnmodifiedSince',
  168. 'CopySourceSSECustomerAlgorithm',
  169. 'CopySourceSSECustomerKey',
  170. 'CopySourceSSECustomerKeyMD5',
  171. 'MetadataDirective'
  172. ]
  173. ALLOWED_DELETE_ARGS = [
  174. 'MFA',
  175. 'VersionId',
  176. 'RequestPayer',
  177. ]
  178. def __init__(self, client, config=None, osutil=None, executor_cls=None):
  179. """A transfer manager interface for Amazon S3
  180. :param client: Client to be used by the manager
  181. :param config: TransferConfig to associate specific configurations
  182. :param osutil: OSUtils object to use for os-related behavior when
  183. using with transfer manager.
  184. :type executor_cls: s3transfer.futures.BaseExecutor
  185. :param executor_cls: The class of executor to use with the transfer
  186. manager. By default, concurrent.futures.ThreadPoolExecutor is used.
  187. """
  188. self._client = client
  189. self._config = config
  190. if config is None:
  191. self._config = TransferConfig()
  192. self._osutil = osutil
  193. if osutil is None:
  194. self._osutil = OSUtils()
  195. self._coordinator_controller = TransferCoordinatorController()
  196. # A counter to create unique id's for each transfer submitted.
  197. self._id_counter = 0
  198. # The executor responsible for making S3 API transfer requests
  199. self._request_executor = BoundedExecutor(
  200. max_size=self._config.max_request_queue_size,
  201. max_num_threads=self._config.max_request_concurrency,
  202. tag_semaphores={
  203. IN_MEMORY_UPLOAD_TAG: TaskSemaphore(
  204. self._config.max_in_memory_upload_chunks),
  205. IN_MEMORY_DOWNLOAD_TAG: SlidingWindowSemaphore(
  206. self._config.max_in_memory_download_chunks)
  207. },
  208. executor_cls=executor_cls
  209. )
  210. # The executor responsible for submitting the necessary tasks to
  211. # perform the desired transfer
  212. self._submission_executor = BoundedExecutor(
  213. max_size=self._config.max_submission_queue_size,
  214. max_num_threads=self._config.max_submission_concurrency,
  215. executor_cls=executor_cls
  216. )
  217. # There is one thread available for writing to disk. It will handle
  218. # downloads for all files.
  219. self._io_executor = BoundedExecutor(
  220. max_size=self._config.max_io_queue_size,
  221. max_num_threads=1,
  222. executor_cls=executor_cls
  223. )
  224. # The component responsible for limiting bandwidth usage if it
  225. # is configured.
  226. self._bandwidth_limiter = None
  227. if self._config.max_bandwidth is not None:
  228. logger.debug(
  229. 'Setting max_bandwidth to %s', self._config.max_bandwidth)
  230. leaky_bucket = LeakyBucket(self._config.max_bandwidth)
  231. self._bandwidth_limiter = BandwidthLimiter(leaky_bucket)
  232. self._register_handlers()
  233. def upload(self, fileobj, bucket, key, extra_args=None, subscribers=None):
  234. """Uploads a file to S3
  235. :type fileobj: str or seekable file-like object
  236. :param fileobj: The name of a file to upload or a seekable file-like
  237. object to upload. It is recommended to use a filename because
  238. file-like objects may result in higher memory usage.
  239. :type bucket: str
  240. :param bucket: The name of the bucket to upload to
  241. :type key: str
  242. :param key: The name of the key to upload to
  243. :type extra_args: dict
  244. :param extra_args: Extra arguments that may be passed to the
  245. client operation
  246. :type subscribers: list(s3transfer.subscribers.BaseSubscriber)
  247. :param subscribers: The list of subscribers to be invoked in the
  248. order provided based on the event emit during the process of
  249. the transfer request.
  250. :rtype: s3transfer.futures.TransferFuture
  251. :returns: Transfer future representing the upload
  252. """
  253. if extra_args is None:
  254. extra_args = {}
  255. if subscribers is None:
  256. subscribers = []
  257. self._validate_all_known_args(extra_args, self.ALLOWED_UPLOAD_ARGS)
  258. call_args = CallArgs(
  259. fileobj=fileobj, bucket=bucket, key=key, extra_args=extra_args,
  260. subscribers=subscribers
  261. )
  262. extra_main_kwargs = {}
  263. if self._bandwidth_limiter:
  264. extra_main_kwargs['bandwidth_limiter'] = self._bandwidth_limiter
  265. return self._submit_transfer(
  266. call_args, UploadSubmissionTask, extra_main_kwargs)
  267. def download(self, bucket, key, fileobj, extra_args=None,
  268. subscribers=None):
  269. """Downloads a file from S3
  270. :type bucket: str
  271. :param bucket: The name of the bucket to download from
  272. :type key: str
  273. :param key: The name of the key to download from
  274. :type fileobj: str
  275. :param fileobj: The name of a file to download to.
  276. :type extra_args: dict
  277. :param extra_args: Extra arguments that may be passed to the
  278. client operation
  279. :type subscribers: list(s3transfer.subscribers.BaseSubscriber)
  280. :param subscribers: The list of subscribers to be invoked in the
  281. order provided based on the event emit during the process of
  282. the transfer request.
  283. :rtype: s3transfer.futures.TransferFuture
  284. :returns: Transfer future representing the download
  285. """
  286. if extra_args is None:
  287. extra_args = {}
  288. if subscribers is None:
  289. subscribers = []
  290. self._validate_all_known_args(extra_args, self.ALLOWED_DOWNLOAD_ARGS)
  291. call_args = CallArgs(
  292. bucket=bucket, key=key, fileobj=fileobj, extra_args=extra_args,
  293. subscribers=subscribers
  294. )
  295. extra_main_kwargs = {'io_executor': self._io_executor}
  296. if self._bandwidth_limiter:
  297. extra_main_kwargs['bandwidth_limiter'] = self._bandwidth_limiter
  298. return self._submit_transfer(
  299. call_args, DownloadSubmissionTask, extra_main_kwargs)
  300. def copy(self, copy_source, bucket, key, extra_args=None,
  301. subscribers=None, source_client=None):
  302. """Copies a file in S3
  303. :type copy_source: dict
  304. :param copy_source: The name of the source bucket, key name of the
  305. source object, and optional version ID of the source object. The
  306. dictionary format is:
  307. ``{'Bucket': 'bucket', 'Key': 'key', 'VersionId': 'id'}``. Note
  308. that the ``VersionId`` key is optional and may be omitted.
  309. :type bucket: str
  310. :param bucket: The name of the bucket to copy to
  311. :type key: str
  312. :param key: The name of the key to copy to
  313. :type extra_args: dict
  314. :param extra_args: Extra arguments that may be passed to the
  315. client operation
  316. :type subscribers: a list of subscribers
  317. :param subscribers: The list of subscribers to be invoked in the
  318. order provided based on the event emit during the process of
  319. the transfer request.
  320. :type source_client: botocore or boto3 Client
  321. :param source_client: The client to be used for operation that
  322. may happen at the source object. For example, this client is
  323. used for the head_object that determines the size of the copy.
  324. If no client is provided, the transfer manager's client is used
  325. as the client for the source object.
  326. :rtype: s3transfer.futures.TransferFuture
  327. :returns: Transfer future representing the copy
  328. """
  329. if extra_args is None:
  330. extra_args = {}
  331. if subscribers is None:
  332. subscribers = []
  333. if source_client is None:
  334. source_client = self._client
  335. self._validate_all_known_args(extra_args, self.ALLOWED_COPY_ARGS)
  336. call_args = CallArgs(
  337. copy_source=copy_source, bucket=bucket, key=key,
  338. extra_args=extra_args, subscribers=subscribers,
  339. source_client=source_client
  340. )
  341. return self._submit_transfer(call_args, CopySubmissionTask)
  342. def delete(self, bucket, key, extra_args=None, subscribers=None):
  343. """Delete an S3 object.
  344. :type bucket: str
  345. :param bucket: The name of the bucket.
  346. :type key: str
  347. :param key: The name of the S3 object to delete.
  348. :type extra_args: dict
  349. :param extra_args: Extra arguments that may be passed to the
  350. DeleteObject call.
  351. :type subscribers: list
  352. :param subscribers: A list of subscribers to be invoked during the
  353. process of the transfer request. Note that the ``on_progress``
  354. callback is not invoked during object deletion.
  355. :rtype: s3transfer.futures.TransferFuture
  356. :return: Transfer future representing the deletion.
  357. """
  358. if extra_args is None:
  359. extra_args = {}
  360. if subscribers is None:
  361. subscribers = []
  362. self._validate_all_known_args(extra_args, self.ALLOWED_DELETE_ARGS)
  363. call_args = CallArgs(
  364. bucket=bucket, key=key, extra_args=extra_args,
  365. subscribers=subscribers
  366. )
  367. return self._submit_transfer(call_args, DeleteSubmissionTask)
  368. def _validate_all_known_args(self, actual, allowed):
  369. for kwarg in actual:
  370. if kwarg not in allowed:
  371. raise ValueError(
  372. "Invalid extra_args key '%s', "
  373. "must be one of: %s" % (
  374. kwarg, ', '.join(allowed)))
  375. def _submit_transfer(self, call_args, submission_task_cls,
  376. extra_main_kwargs=None):
  377. if not extra_main_kwargs:
  378. extra_main_kwargs = {}
  379. # Create a TransferFuture to return back to the user
  380. transfer_future, components = self._get_future_with_components(
  381. call_args)
  382. # Add any provided done callbacks to the created transfer future
  383. # to be invoked on the transfer future being complete.
  384. for callback in get_callbacks(transfer_future, 'done'):
  385. components['coordinator'].add_done_callback(callback)
  386. # Get the main kwargs needed to instantiate the submission task
  387. main_kwargs = self._get_submission_task_main_kwargs(
  388. transfer_future, extra_main_kwargs)
  389. # Submit a SubmissionTask that will submit all of the necessary
  390. # tasks needed to complete the S3 transfer.
  391. self._submission_executor.submit(
  392. submission_task_cls(
  393. transfer_coordinator=components['coordinator'],
  394. main_kwargs=main_kwargs
  395. )
  396. )
  397. # Increment the unique id counter for future transfer requests
  398. self._id_counter += 1
  399. return transfer_future
  400. def _get_future_with_components(self, call_args):
  401. transfer_id = self._id_counter
  402. # Creates a new transfer future along with its components
  403. transfer_coordinator = TransferCoordinator(transfer_id=transfer_id)
  404. # Track the transfer coordinator for transfers to manage.
  405. self._coordinator_controller.add_transfer_coordinator(
  406. transfer_coordinator)
  407. # Also make sure that the transfer coordinator is removed once
  408. # the transfer completes so it does not stick around in memory.
  409. transfer_coordinator.add_done_callback(
  410. self._coordinator_controller.remove_transfer_coordinator,
  411. transfer_coordinator)
  412. components = {
  413. 'meta': TransferMeta(call_args, transfer_id=transfer_id),
  414. 'coordinator': transfer_coordinator
  415. }
  416. transfer_future = TransferFuture(**components)
  417. return transfer_future, components
  418. def _get_submission_task_main_kwargs(
  419. self, transfer_future, extra_main_kwargs):
  420. main_kwargs = {
  421. 'client': self._client,
  422. 'config': self._config,
  423. 'osutil': self._osutil,
  424. 'request_executor': self._request_executor,
  425. 'transfer_future': transfer_future
  426. }
  427. main_kwargs.update(extra_main_kwargs)
  428. return main_kwargs
  429. def _register_handlers(self):
  430. # Register handlers to enable/disable callbacks on uploads.
  431. event_name = 'request-created.s3'
  432. self._client.meta.events.register_first(
  433. event_name, signal_not_transferring,
  434. unique_id='s3upload-not-transferring')
  435. self._client.meta.events.register_last(
  436. event_name, signal_transferring,
  437. unique_id='s3upload-transferring')
  438. def __enter__(self):
  439. return self
  440. def __exit__(self, exc_type, exc_value, *args):
  441. cancel = False
  442. cancel_msg = ''
  443. cancel_exc_type = FatalError
  444. # If a exception was raised in the context handler, signal to cancel
  445. # all of the inprogress futures in the shutdown.
  446. if exc_type:
  447. cancel = True
  448. cancel_msg = six.text_type(exc_value)
  449. if not cancel_msg:
  450. cancel_msg = repr(exc_value)
  451. # If it was a KeyboardInterrupt, the cancellation was initiated
  452. # by the user.
  453. if isinstance(exc_value, KeyboardInterrupt):
  454. cancel_exc_type = CancelledError
  455. self._shutdown(cancel, cancel_msg, cancel_exc_type)
  456. def shutdown(self, cancel=False, cancel_msg=''):
  457. """Shutdown the TransferManager
  458. It will wait till all transfers complete before it completely shuts
  459. down.
  460. :type cancel: boolean
  461. :param cancel: If True, calls TransferFuture.cancel() for
  462. all in-progress in transfers. This is useful if you want the
  463. shutdown to happen quicker.
  464. :type cancel_msg: str
  465. :param cancel_msg: The message to specify if canceling all in-progress
  466. transfers.
  467. """
  468. self._shutdown(cancel, cancel, cancel_msg)
  469. def _shutdown(self, cancel, cancel_msg, exc_type=CancelledError):
  470. if cancel:
  471. # Cancel all in-flight transfers if requested, before waiting
  472. # for them to complete.
  473. self._coordinator_controller.cancel(cancel_msg, exc_type)
  474. try:
  475. # Wait until there are no more in-progress transfers. This is
  476. # wrapped in a try statement because this can be interrupted
  477. # with a KeyboardInterrupt that needs to be caught.
  478. self._coordinator_controller.wait()
  479. except KeyboardInterrupt:
  480. # If not errors were raised in the try block, the cancel should
  481. # have no coordinators it needs to run cancel on. If there was
  482. # an error raised in the try statement we want to cancel all of
  483. # the inflight transfers before shutting down to speed that
  484. # process up.
  485. self._coordinator_controller.cancel('KeyboardInterrupt()')
  486. raise
  487. finally:
  488. # Shutdown all of the executors.
  489. self._submission_executor.shutdown()
  490. self._request_executor.shutdown()
  491. self._io_executor.shutdown()
  492. class TransferCoordinatorController(object):
  493. def __init__(self):
  494. """Abstraction to control all transfer coordinators
  495. This abstraction allows the manager to wait for inprogress transfers
  496. to complete and cancel all inprogress transfers.
  497. """
  498. self._lock = threading.Lock()
  499. self._tracked_transfer_coordinators = set()
  500. @property
  501. def tracked_transfer_coordinators(self):
  502. """The set of transfer coordinators being tracked"""
  503. with self._lock:
  504. # We return a copy because the set is mutable and if you were to
  505. # iterate over the set, it may be changing in length due to
  506. # additions and removals of transfer coordinators.
  507. return copy.copy(self._tracked_transfer_coordinators)
  508. def add_transfer_coordinator(self, transfer_coordinator):
  509. """Adds a transfer coordinator of a transfer to be canceled if needed
  510. :type transfer_coordinator: s3transfer.futures.TransferCoordinator
  511. :param transfer_coordinator: The transfer coordinator for the
  512. particular transfer
  513. """
  514. with self._lock:
  515. self._tracked_transfer_coordinators.add(transfer_coordinator)
  516. def remove_transfer_coordinator(self, transfer_coordinator):
  517. """Remove a transfer coordinator from cancelation consideration
  518. Typically, this method is invoked by the transfer coordinator itself
  519. to remove its self when it completes its transfer.
  520. :type transfer_coordinator: s3transfer.futures.TransferCoordinator
  521. :param transfer_coordinator: The transfer coordinator for the
  522. particular transfer
  523. """
  524. with self._lock:
  525. self._tracked_transfer_coordinators.remove(transfer_coordinator)
  526. def cancel(self, msg='', exc_type=CancelledError):
  527. """Cancels all inprogress transfers
  528. This cancels the inprogress transfers by calling cancel() on all
  529. tracked transfer coordinators.
  530. :param msg: The message to pass on to each transfer coordinator that
  531. gets cancelled.
  532. :param exc_type: The type of exception to set for the cancellation
  533. """
  534. for transfer_coordinator in self.tracked_transfer_coordinators:
  535. transfer_coordinator.cancel(msg, exc_type)
  536. def wait(self):
  537. """Wait until there are no more inprogress transfers
  538. This will not stop when failures are encountered and not propogate any
  539. of these errors from failed transfers, but it can be interrupted with
  540. a KeyboardInterrupt.
  541. """
  542. try:
  543. transfer_coordinator = None
  544. for transfer_coordinator in self.tracked_transfer_coordinators:
  545. transfer_coordinator.result()
  546. except KeyboardInterrupt:
  547. logger.debug('Received KeyboardInterrupt in wait()')
  548. # If Keyboard interrupt is raised while waiting for
  549. # the result, then exit out of the wait and raise the
  550. # exception
  551. if transfer_coordinator:
  552. logger.debug(
  553. 'On KeyboardInterrupt was waiting for %s',
  554. transfer_coordinator)
  555. raise
  556. except Exception:
  557. # A general exception could have been thrown because
  558. # of result(). We just want to ignore this and continue
  559. # because we at least know that the transfer coordinator
  560. # has completed.
  561. pass