1616
1717import collections
1818import functools
19+ import itertools
1920import logging
2021import threading
2122import uuid
3637from google .pubsub_v1 import types as gapic_types
3738
3839_LOGGER = logging .getLogger (__name__ )
40+ _REGULAR_SHUTDOWN_THREAD_NAME = "Thread-RegularStreamShutdown"
3941_RPC_ERROR_THREAD_NAME = "Thread-OnRpcTerminated"
4042_RETRYABLE_STREAM_ERRORS = (
4143 exceptions .DeadlineExceeded ,
@@ -110,24 +112,34 @@ class StreamingPullManager(object):
110112 scheduler (~google.cloud.pubsub_v1.scheduler.Scheduler): The scheduler
111113 to use to process messages. If not provided, a thread pool-based
112114 scheduler will be used.
115+ await_callbacks_on_shutdown (bool):
116+ If ``True``, the shutdown thread will wait until all scheduler threads
117+ terminate and only then proceed with shutting down the remaining running
118+ helper threads.
119+
120+ If ``False`` (default), the shutdown thread will shut the scheduler down,
121+ but it will not wait for the currently executing scheduler threads to
122+ terminate.
123+
124+ This setting affects when the on close callbacks get invoked, and
125+ consequently, when the StreamingPullFuture associated with the stream gets
126+ resolved.
113127 """
114128
115- _UNARY_REQUESTS = True
116- """If set to True, this class will make requests over a separate unary
117- RPC instead of over the streaming RPC."""
118-
119129 def __init__ (
120130 self ,
121131 client ,
122132 subscription ,
123133 flow_control = types .FlowControl (),
124134 scheduler = None ,
125135 use_legacy_flow_control = False ,
136+ await_callbacks_on_shutdown = False ,
126137 ):
127138 self ._client = client
128139 self ._subscription = subscription
129140 self ._flow_control = flow_control
130141 self ._use_legacy_flow_control = use_legacy_flow_control
142+ self ._await_callbacks_on_shutdown = await_callbacks_on_shutdown
131143 self ._ack_histogram = histogram .Histogram ()
132144 self ._last_histogram_size = 0
133145 self ._ack_deadline = 10
@@ -291,6 +303,9 @@ def activate_ordering_keys(self, ordering_keys):
291303 activate. May be empty.
292304 """
293305 with self ._pause_resume_lock :
306+ if self ._scheduler is None :
307+ return # We are shutting down, don't try to dispatch any more messages.
308+
294309 self ._messages_on_hold .activate_ordering_keys (
295310 ordering_keys , self ._schedule_message_on_hold
296311 )
@@ -420,37 +435,36 @@ def send(self, request):
420435 If a RetryError occurs, the manager shutdown is triggered, and the
421436 error is re-raised.
422437 """
423- if self ._UNARY_REQUESTS :
424- try :
425- self ._send_unary_request (request )
426- except exceptions .GoogleAPICallError :
427- _LOGGER .debug (
428- "Exception while sending unary RPC. This is typically "
429- "non-fatal as stream requests are best-effort." ,
430- exc_info = True ,
431- )
432- except exceptions .RetryError as exc :
433- _LOGGER .debug (
434- "RetryError while sending unary RPC. Waiting on a transient "
435- "error resolution for too long, will now trigger shutdown." ,
436- exc_info = False ,
437- )
438- # The underlying channel has been suffering from a retryable error
439- # for too long, time to give up and shut the streaming pull down.
440- self ._on_rpc_done (exc )
441- raise
442-
443- else :
444- self ._rpc .send (request )
438+ try :
439+ self ._send_unary_request (request )
440+ except exceptions .GoogleAPICallError :
441+ _LOGGER .debug (
442+ "Exception while sending unary RPC. This is typically "
443+ "non-fatal as stream requests are best-effort." ,
444+ exc_info = True ,
445+ )
446+ except exceptions .RetryError as exc :
447+ _LOGGER .debug (
448+ "RetryError while sending unary RPC. Waiting on a transient "
449+ "error resolution for too long, will now trigger shutdown." ,
450+ exc_info = False ,
451+ )
452+ # The underlying channel has been suffering from a retryable error
453+ # for too long, time to give up and shut the streaming pull down.
454+ self ._on_rpc_done (exc )
455+ raise
445456
446457 def heartbeat (self ):
447458 """Sends an empty request over the streaming pull RPC.
448459
449- This always sends over the stream, regardless of if
450- ``self._UNARY_REQUESTS`` is set or not .
460+ Returns:
461+ bool: If a heartbeat request has actually been sent .
451462 """
452463 if self ._rpc is not None and self ._rpc .is_active :
453464 self ._rpc .send (gapic_types .StreamingPullRequest ())
465+ return True
466+
467+ return False
454468
455469 def open (self , callback , on_callback_error ):
456470 """Begin consuming messages.
@@ -517,11 +531,29 @@ def close(self, reason=None):
517531
518532 This method is idempotent. Additional calls will have no effect.
519533
534+ The method does not block, it delegates the shutdown operations to a background
535+ thread.
536+
520537 Args:
521- reason (Any): The reason to close this. If None, this is considered
538+ reason (Any): The reason to close this. If `` None`` , this is considered
522539 an "intentional" shutdown. This is passed to the callbacks
523540 specified via :meth:`add_close_callback`.
524541 """
542+ thread = threading .Thread (
543+ name = _REGULAR_SHUTDOWN_THREAD_NAME ,
544+ daemon = True ,
545+ target = self ._shutdown ,
546+ kwargs = {"reason" : reason },
547+ )
548+ thread .start ()
549+
550+ def _shutdown (self , reason = None ):
551+ """Run the actual shutdown sequence (stop the stream and all helper threads).
552+
553+ Args:
554+ reason (Any): The reason to close the stream. If ``None``, this is
555+ considered an "intentional" shutdown.
556+ """
525557 with self ._closing :
526558 if self ._closed :
527559 return
@@ -534,7 +566,9 @@ def close(self, reason=None):
534566
535567 # Shutdown all helper threads
536568 _LOGGER .debug ("Stopping scheduler." )
537- self ._scheduler .shutdown ()
569+ dropped_messages = self ._scheduler .shutdown (
570+ await_msg_callbacks = self ._await_callbacks_on_shutdown
571+ )
538572 self ._scheduler = None
539573
540574 # Leaser and dispatcher reference each other through the shared
@@ -548,11 +582,23 @@ def close(self, reason=None):
548582 # because the consumer gets shut down first.
549583 _LOGGER .debug ("Stopping leaser." )
550584 self ._leaser .stop ()
585+
586+ total = len (dropped_messages ) + len (
587+ self ._messages_on_hold ._messages_on_hold
588+ )
589+ _LOGGER .debug (f"NACK-ing all not-yet-dispatched messages (total: { total } )." )
590+ messages_to_nack = itertools .chain (
591+ dropped_messages , self ._messages_on_hold ._messages_on_hold
592+ )
593+ for msg in messages_to_nack :
594+ msg .nack ()
595+
551596 _LOGGER .debug ("Stopping dispatcher." )
552597 self ._dispatcher .stop ()
553598 self ._dispatcher = None
554599 # dispatcher terminated, OK to dispose the leaser reference now
555600 self ._leaser = None
601+
556602 _LOGGER .debug ("Stopping heartbeater." )
557603 self ._heartbeater .stop ()
558604 self ._heartbeater = None
@@ -722,7 +768,7 @@ def _on_rpc_done(self, future):
722768 _LOGGER .info ("RPC termination has signaled streaming pull manager shutdown." )
723769 error = _wrap_as_exception (future )
724770 thread = threading .Thread (
725- name = _RPC_ERROR_THREAD_NAME , target = self .close , kwargs = {"reason" : error }
771+ name = _RPC_ERROR_THREAD_NAME , target = self ._shutdown , kwargs = {"reason" : error }
726772 )
727773 thread .daemon = True
728774 thread .start ()
0 commit comments