From fed848c276656992be63ea26f5e4146e5aacca22 Mon Sep 17 00:00:00 2001 From: annatisch Date: Mon, 18 Jun 2018 10:13:05 +1200 Subject: [PATCH 01/52] Added warning for Python 2.7 support Support for issues #36 and #38 --- README.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.rst b/README.rst index 4641d34..616771e 100644 --- a/README.rst +++ b/README.rst @@ -21,6 +21,10 @@ Wheels are provided for all major operating systems, so you can install directly $ pip install azure-eventhub +Python 2.7 support +++++++++++++++++++ +The uAMQP library currently only supports Python 3.4 and above. Python 2.7 support is planned for a future release. + Examples +++++++++ From ed2df7b24c47f83d986bc759155e06473f2b1c71 Mon Sep 17 00:00:00 2001 From: annatisch Date: Mon, 25 Jun 2018 08:15:20 -0700 Subject: [PATCH 02/52] Started adding scenario tests --- features/eph.feature | 21 ++++++++++++ features/eventhub.feature | 67 ++++++++++++++++++++++++++++++++++++++ features/steps/eventhub.py | 13 ++++++++ 3 files changed, 101 insertions(+) create mode 100644 features/eph.feature create mode 100644 features/eventhub.feature create mode 100644 features/steps/eventhub.py diff --git a/features/eph.feature b/features/eph.feature new file mode 100644 index 0000000..2742eaf --- /dev/null +++ b/features/eph.feature @@ -0,0 +1,21 @@ +Feature: Exercising Event Processor Host + +# Scenario: EPH single host, generic scenario. + +# Scenario: EPH runs with listen only claims. + +# Scenario: Host runs idle for a while by managing sender to send in intervals. + +# Scenario: No sends at all, hosts will stay idle. + +# Scenario: Spawns multiple test processes consuming from the same event hub. + +# Scenario: Registers and unregisters hosts as part of the regular iteration to introduce excessive partition moves. + + Scenario: Registers and unregisters hosts as part of the regular iteration to introduce excessive partition moves. No sends in this scenario. + + Scenario: Runs EPH on 256 partition entity. + +# Scenario: Runs EPH on multiple consumer groups. + +# Scenario: Runs EPH with web sockets enabled. \ No newline at end of file diff --git a/features/eventhub.feature b/features/eventhub.feature new file mode 100644 index 0000000..db09254 --- /dev/null +++ b/features/eventhub.feature @@ -0,0 +1,67 @@ +Feature: Exercising EventHub SDK + +# Scenario: Just sends for 3 days, no receives. Focus on send failures only. + + @long-running + Scenario: Generic send and receive on client for 3 days. + Given the EventHub SDK is installed + And an EventHub is created with credentials retrieved + When I send and receive messages for 72 hours + Then I should receive no errors + And I can shutdown the sender and receiver cleanly. + +# Scenario: Sender stays idle for 45 minutes and sends some number of messages after each idle duration. + +# Scenario: Sends on partition senders. + +# Scenario: Send and receive to/from a multiple consumer group entity. + +# Scenario: Sends and receives 246KB size messages. + + @long-running + Scenario: Runs on a 100TU namespace and saturates ingress. + Given the EventHub SDK is installed + And an EventHub with 100TU is created with credentials retrieved + When I send messages for 2 hours + Then I should achieve throughput of greater than 3600000 messages + And I should receive no errors + And I can shutdown the sender cleanly. + + @long-running + Scenario: Runs on a 100TU namespace and saturates ingress with partition senders for 3 days. + Given the EventHub SDK is installed + And an EventHub with 100TU is created with credentials retrieved + When I send messages to partitions for 2 hours + Then I should achieve throughput of greater than 1800000 messages to each partition + And I should receive no errors + And I can shutdown the sender cleanly. + +# Scenario: Sends and receives 1 byte size messages. + +# Scenario: Single clients parks 500 async sends. + +# Scenario: Sends a set of messages and keeps receiving same set of messages again and again. + +# Scenario: Receives with 60 minutes of receive timeout. + +# Scenario: Receives with 3 seconds of receive timeout. + +# Scenario: Recreates receivers at the beginning of each iteration. + +# Scenario: Recreates receivers with the last known sequence number at the beginning of each iteration. + +# Scenario: Uses epoch receivers. + +# Scenario: Introduces a short idle time after each receive attempt. We use 50 seconds of sleep here. + +# Scenario: Uses pump receivers to receive messages. + +# Scenario: Sends messages with partition key set. + +# Scenario: Issues runtime information API calls as part of send and receive. + +# Scenario: Uses batch sender to send messages. + +# Scenario: Sends and receives by enabling web sockets over AMQP. + +# Scenario: Issues runtime information API calls over web sockets as part of send and receive. diff --git a/features/steps/eventhub.py b/features/steps/eventhub.py new file mode 100644 index 0000000..1e2341d --- /dev/null +++ b/features/steps/eventhub.py @@ -0,0 +1,13 @@ +from behave import * + +@given('we have behave installed') +def step_impl(context): + pass + +@when('we implement a test') +def step_impl(context): + assert True is not False + +@then('behave will test it for us!') +def step_impl(context): + assert context.failed is False \ No newline at end of file From e57776d016c08c6a454557290bda6a6aba830bee Mon Sep 17 00:00:00 2001 From: annatisch Date: Mon, 25 Jun 2018 10:18:46 -0700 Subject: [PATCH 03/52] More test scenarios --- .gitignore | 6 +++ azure/eventhub/__init__.py | 2 +- azure/eventprocessorhost/partition_context.py | 6 ++- azure/eventprocessorhost/partition_pump.py | 2 +- dev_requirements.txt | 3 +- features/eph.feature | 5 ++ features/eventhub.feature | 16 ++++-- features/steps/eventhub.py | 50 +++++++++++++++++-- features/steps/test_utils.py | 34 +++++++++++++ setup.py | 2 +- 10 files changed, 112 insertions(+), 14 deletions(-) create mode 100644 features/steps/test_utils.py diff --git a/.gitignore b/.gitignore index 27e1a0e..ef97309 100644 --- a/.gitignore +++ b/.gitignore @@ -105,3 +105,9 @@ ENV/ .mypy_cache/ .pytest_cache/v/cache/lastfailed .pytest_cache/v/cache/nodeids + +# EventHub +azure/mgmt/ +azure/common/ +azure/profiles/ +features/steps/mgmt_settings_real.py diff --git a/azure/eventhub/__init__.py b/azure/eventhub/__init__.py index 8ad8448..072de4d 100644 --- a/azure/eventhub/__init__.py +++ b/azure/eventhub/__init__.py @@ -430,7 +430,7 @@ def on_message(self, event): Callback to process a received message and wrap it in EventData. Will also call a user supplied callback. :param event: The received message. - :type event: ~uamqp.Message + :type event: ~uamqp.message.Message :returns: ~azure.eventhub.EventData. """ event_data = EventData(message=event) diff --git a/azure/eventprocessorhost/partition_context.py b/azure/eventprocessorhost/partition_context.py index 4858adc..d4cfa19 100644 --- a/azure/eventprocessorhost/partition_context.py +++ b/azure/eventprocessorhost/partition_context.py @@ -88,6 +88,7 @@ def to_string(self): """ Returns the parition context in the following format: "PartitionContext({EventHubPath}{ConsumerGroupName}{PartitionId}{SequenceNumber})" + :returns: str """ return "PartitionContext({}{}{}{})".format(self.eh_path, @@ -97,9 +98,10 @@ def to_string(self): async def persist_checkpoint_async(self, checkpoint): """ - Persists the checkpoint + Persists the checkpoint. + :param checkpoint: The checkpoint to persist. - :type checkpoint: ~azure.eventprocessorhost.Checkpoint + :type checkpoint: ~azure.eventprocessorhost.checkpoint.Checkpoint """ _logger.debug("PartitionPumpCheckpointStart {} {} {} {}".format( self.host.guid, checkpoint.partition_id, checkpoint.offset, checkpoint.sequence_number)) diff --git a/azure/eventprocessorhost/partition_pump.py b/azure/eventprocessorhost/partition_pump.py index 6caa981..1327463 100644 --- a/azure/eventprocessorhost/partition_pump.py +++ b/azure/eventprocessorhost/partition_pump.py @@ -43,7 +43,7 @@ def set_lease(self, new_lease): """ Sets a new partition lease to be processed by the pump :param lease: The lease to set. - :type lease: ~azure.eventprocessorhost.Lease + :type lease: ~azure.eventprocessorhost.lease.Lease """ if self.partition_context: self.partition_context.lease = new_lease diff --git a/dev_requirements.txt b/dev_requirements.txt index a6bbee0..3cbeb9a 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -2,4 +2,5 @@ pytest>=3.4.1 pytest-asyncio>=0.8.0 docutils>=0.14 pygments>=2.2.0 -pylint==1.8.4 \ No newline at end of file +pylint==1.8.4 +behave==1.2.6 \ No newline at end of file diff --git a/features/eph.feature b/features/eph.feature index 2742eaf..73fc8bc 100644 --- a/features/eph.feature +++ b/features/eph.feature @@ -1,3 +1,8 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + Feature: Exercising Event Processor Host # Scenario: EPH single host, generic scenario. diff --git a/features/eventhub.feature b/features/eventhub.feature index db09254..c96a998 100644 --- a/features/eventhub.feature +++ b/features/eventhub.feature @@ -1,3 +1,8 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + Feature: Exercising EventHub SDK # Scenario: Just sends for 3 days, no receives. Focus on send failures only. @@ -8,7 +13,8 @@ Feature: Exercising EventHub SDK And an EventHub is created with credentials retrieved When I send and receive messages for 72 hours Then I should receive no errors - And I can shutdown the sender and receiver cleanly. + And I can shutdown the sender and receiver cleanly + And I remove the EventHub # Scenario: Sender stays idle for 45 minutes and sends some number of messages after each idle duration. @@ -25,16 +31,18 @@ Feature: Exercising EventHub SDK When I send messages for 2 hours Then I should achieve throughput of greater than 3600000 messages And I should receive no errors - And I can shutdown the sender cleanly. + And I can shutdown the sender cleanly + And I remove the EventHub @long-running Scenario: Runs on a 100TU namespace and saturates ingress with partition senders for 3 days. Given the EventHub SDK is installed And an EventHub with 100TU is created with credentials retrieved When I send messages to partitions for 2 hours - Then I should achieve throughput of greater than 1800000 messages to each partition + Then I should achieve throughput of greater than 1800000 messages from each partition And I should receive no errors - And I can shutdown the sender cleanly. + And I can shutdown the sender cleanly + And I remove the EventHub # Scenario: Sends and receives 1 byte size messages. diff --git a/features/steps/eventhub.py b/features/steps/eventhub.py index 1e2341d..b92bdc5 100644 --- a/features/steps/eventhub.py +++ b/features/steps/eventhub.py @@ -1,13 +1,55 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import asyncio +import uuid + from behave import * -@given('we have behave installed') +import test_utils + +@given('the EventHub SDK is installed') def step_impl(context): - pass + from azure import eventhub -@when('we implement a test') +@given('an EventHub is created with credentials retrieved') def step_impl(context): + #from mgmt_settings_real import get_credentials, SUBSCRIPTION_ID + #rg, mgmt_client = test_utils.create_mgmt_client(get_credentials(), SUBSCRIPTION_ID) + context.eh_config = test_utils.get_eventhub_config() + +@given('an EventHub with {properties} is created with credentials retrieved') +def step_impl(context, properties): + #from mgmt_settings_real import get_credentials, SUBSCRIPTION_ID + #rg, mgmt_client = test_utils.create_mgmt_client(get_credentials(), SUBSCRIPTION_ID) + context.eh_config = test_utils.get_eventhub_config() + +@when('I {clients} messages for {hours} hours') +def step_impl(context, clients, hours): + assert True is not False + +@when('I {clients} messages {destination} for {hours} hours') +def step_impl(context, clients, destination, hours): assert True is not False -@then('behave will test it for us!') +@then('I should receive no errors') +def step_impl(context): + assert context.failed is False + +@then('I can shutdown the {clients} cleanly') +def step_impl(context, clients): + assert context.failed is False + +@then('I should achieve throughput of greater than {total} messages') +def step_impl(context, total): + assert context.failed is False + +@then('I should achieve throughput of greater than {total} messages from {source}') +def step_impl(context, total, source): + assert context.failed is False + +@then('I remove the EventHub') def step_impl(context): assert context.failed is False \ No newline at end of file diff --git a/features/steps/test_utils.py b/features/steps/test_utils.py new file mode 100644 index 0000000..31eb7b5 --- /dev/null +++ b/features/steps/test_utils.py @@ -0,0 +1,34 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import uuid + +def create_mgmt_client(credentials, subscription, location='westus'): + from azure.mgmt.resource import ResourceManagementClient + from azure.mgmt.eventhub import EventHubManagementClient + + resource_client = ResourceManagementClient(credentials, subscription) + rg_name = 'pytest-{}'.format(uuid.uuid4()) + resource_group = resource_client.resource_groups.create_or_update( + rg_name, {'location': location}) + + eh_client = EventHubManagementClient(credentials, subscription) + namespace = 'pytest-{}'.format(uuid.uuid4()) + creator = eh_client.namespaces.create_or_update( + resource_group.name, + namespace) + create.wait() + return resource_group, eh_client + + +def get_eventhub_config(): + config = {} + config['hostname'] = os.environ['EVENT_HUB_HOSTNAME'] + config['event_hub'] = os.environ['EVENT_HUB_NAME'] + config['key_name'] = os.environ['EVENT_HUB_SAS_POLICY'] + config['access_key'] = os.environ['EVENT_HUB_SAS_KEY'] + config['consumer_group'] = "$Default" + config['partition'] = "0" + return config diff --git a/setup.py b/setup.py index 66f5465..1cc16ce 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ zip_safe=False, packages=find_packages(exclude=["examples", "tests"]), install_requires=[ - 'uamqp~=0.1.0rc1', + 'uamqp==0.1.0rc1', 'msrestazure~=0.4.11', 'azure-common~=1.1', 'azure-storage~=0.36.0' From 684f1f3c004e8d8473683a30388a829ee8b78cd0 Mon Sep 17 00:00:00 2001 From: annatisch Date: Mon, 25 Jun 2018 11:21:38 -0700 Subject: [PATCH 04/52] Better docstring formatting --- azure/eventhub/__init__.py | 77 +++++++++++++------ azure/eventhub/async/__init__.py | 36 +++++---- .../abstract_checkpoint_manager.py | 20 +++-- .../abstract_event_processor.py | 16 ++-- .../abstract_lease_manager.py | 47 +++++++---- azure/eventprocessorhost/azure_blob_lease.py | 12 +-- .../azure_storage_checkpoint_manager.py | 75 ++++++++++++++---- .../eventprocessorhost/cancellation_token.py | 4 +- azure/eventprocessorhost/checkpoint.py | 8 +- azure/eventprocessorhost/eh_config.py | 5 ++ azure/eventprocessorhost/eh_partition_pump.py | 21 ++--- azure/eventprocessorhost/eph.py | 19 ++--- azure/eventprocessorhost/lease.py | 11 ++- azure/eventprocessorhost/partition_context.py | 13 ++-- azure/eventprocessorhost/partition_manager.py | 58 ++++++++++---- azure/eventprocessorhost/partition_pump.py | 20 +++-- 16 files changed, 304 insertions(+), 138 deletions(-) diff --git a/azure/eventhub/__init__.py b/azure/eventhub/__init__.py index 072de4d..7892358 100644 --- a/azure/eventhub/__init__.py +++ b/azure/eventhub/__init__.py @@ -106,6 +106,7 @@ def __init__(self, address, username=None, password=None, debug=False): def from_connection_string(cls, conn_str, eventhub=None, **kwargs): """ Create an EventHubClient from a connection string. + :param conn_str: The connection string. :type conn_str: str :param eventhub: The name of the EventHub, if the EntityName is @@ -135,7 +136,7 @@ def _create_properties(self): # pylint: disable=no-self-use Format the properties with which to instantiate the connection. This acts like a user agent over HTTP. - :returns: dict + :rtype: dict """ properties = {} properties["product"] = "eventhub.python" @@ -146,7 +147,7 @@ def _create_properties(self): # pylint: disable=no-self-use def _create_connection(self): """ - Create a new ~uamqp.Connection instance that will be shared between all + Create a new ~uamqp.connection.Connection instance that will be shared between all Sender/Receiver clients. """ if not self.connection: @@ -179,7 +180,7 @@ def run(self): Run the EventHubClient in blocking mode. Opens the connection and starts running all Sender/Receiver clients. - :returns: ~azure.eventhub.EventHubClient + :rtype: ~azure.eventhub.EventHubClient """ log.info("{}: Starting {} clients".format(self.container_id, len(self.clients))) self._create_connection() @@ -205,7 +206,8 @@ def get_eventhub_info(self): -'created_at' -'partition_count' -'partition_ids' - :returns: dict + + :rtype: dict """ self._create_connection() eh_name = self.address.path.lstrip('/') @@ -237,6 +239,7 @@ def get_eventhub_info(self): def add_receiver(self, consumer_group, partition, offset=None, prefetch=300): """ Add a receiver to the client for a particular consumer group and partition. + :param consumer_group: The name of the consumer group. :type consumer_group: str :param partition: The ID of the partition. @@ -245,7 +248,7 @@ def add_receiver(self, consumer_group, partition, offset=None, prefetch=300): :type offset: ~azure.eventhub.Offset :param prefetch: The message prefetch count of the receiver. Default is 300. :type prefetch: int - :returns: ~azure.eventhub.Receiver + :rtype: ~azure.eventhub.Receiver """ source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( self.address.hostname, self.address.path, consumer_group, partition) @@ -262,6 +265,7 @@ def add_epoch_receiver(self, consumer_group, partition, epoch, prefetch=300): can connect to a partition at any given time - additional epoch receivers must have a higher epoch value or they will be rejected. If a 2nd epoch receiver has connected, the first will be closed. + :param consumer_group: The name of the consumer group. :type consumer_group: str :param partition: The ID of the partition. @@ -270,7 +274,7 @@ def add_epoch_receiver(self, consumer_group, partition, epoch, prefetch=300): :type epoch: int :param prefetch: The message prefetch count of the receiver. Default is 300. :type prefetch: int - :returns: ~azure.eventhub.Receiver + :rtype: ~azure.eventhub.Receiver """ source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( self.address.hostname, self.address.path, consumer_group, partition) @@ -282,11 +286,12 @@ def add_sender(self, partition=None): """ Add a sender to the client to send ~azure.eventhub.EventData object to an EventHub. + :param partition: Optionally specify a particular partition to send to. If omitted, the events will be distributed to available partitions via - round-robin + round-robin. :type parition: str - :returns: ~azure.eventhub.Sender + :rtype: ~azure.eventhub.Sender """ target = "amqps://{}{}".format(self.address.hostname, self.address.path) handler = Sender(self, target, partition=partition) @@ -303,6 +308,7 @@ class Sender: def __init__(self, client, target, partition=None): """ Instantiate an EventHub event Sender client. + :param client: The parent EventHubClient. :type client: ~azure.eventhub.EventHubClient. :param target: The URI of the EventHub to send to. @@ -323,11 +329,13 @@ def send(self, event_data): """ Sends an event data and blocks until acknowledgement is received or operation times out. + :param event_data: The event to be sent. :type event_data: ~azure.eventhub.EventData :raises: ~azure.eventhub.EventHubError if the message fails to send. - :returns: The outcome of the message send ~uamqp.constants.MessageSendResult + :return: The outcome of the message send. + :rtype: ~uamqp.constants.MessageSendResult """ if event_data.partition_key and self.partition: raise ValueError("EventData partition key cannot be used with a partition sender.") @@ -344,6 +352,7 @@ def send(self, event_data): def transfer(self, event_data, callback=None): """ Transfers an event data and notifies the callback when the operation is done. + :param event_data: The event to be sent. :type event_data: ~azure.eventhub.EventData :param callback: Callback to be run once the message has been send. @@ -368,6 +377,7 @@ def wait(self): def _on_outcome(self, outcome, condition): """ Called when the outcome is received for a delivery. + :param outcome: The outcome of the message delivery - success or failure. :type outcome: ~uamqp.constants.MessageSendResult """ @@ -389,10 +399,11 @@ class Receiver: def __init__(self, client, source, prefetch=300, epoch=None): """ Instantiate a receiver. + :param client: The parent EventHubClient. :type client: ~azure.eventhub.EventHubClient :param source: The source EventHub from which to receive events. - :type source: ~uamqp.Source + :type source: ~uamqp.address.Source :param prefetch: The number of events to prefetch from the service for processing. Default is 300. :type prefetch: int @@ -418,7 +429,8 @@ def __init__(self, client, source, prefetch=300, epoch=None): def queue_size(self): """ The current size of the unprocessed message queue. - :returns: int + + :rtype: int """ # pylint: disable=protected-access if self._handler._received_messages: @@ -429,9 +441,10 @@ def on_message(self, event): """ Callback to process a received message and wrap it in EventData. Will also call a user supplied callback. + :param event: The received message. :type event: ~uamqp.message.Message - :returns: ~azure.eventhub.EventData. + :rtype: ~azure.eventhub.EventData. """ event_data = EventData(message=event) if self._callback: @@ -442,6 +455,7 @@ def on_message(self, event): def receive(self, max_batch_size=None, callback=None, timeout=None): """ Receive events from the EventHub. + :param max_batch_size: Receive a batch of events. Batch size will be up to the maximum specified, but will return as soon as service returns no new events. If combined with a timeout and no events are @@ -452,7 +466,7 @@ def receive(self, max_batch_size=None, callback=None, timeout=None): be a function that accepts a single argument - the event data. This callback will be run before the message is returned in the result generator. :type callback: func[~azure.eventhub.EventData] - :returns: list[~azure.eventhub.EventData] + :rtype: list[~azure.eventhub.EventData] """ try: timeout_ms = 1000 * timeout if timeout else 0 @@ -478,9 +492,10 @@ def receive(self, max_batch_size=None, callback=None, timeout=None): def selector(self, default): """ Create a selector for the current offset if it is set. + :param default: The fallback receive offset. :type default: ~azure.eventhub.Offset - :returns: ~azure.eventhub.Offset + :rtype: ~azure.eventhub.Offset """ if self.offset is not None: return Offset(self.offset).selector() @@ -490,7 +505,7 @@ def selector(self, default): class EventData(object): """ The EventData class is a holder of event content. - Acts as a wrapper to an ~uamqp.Message object. + Acts as a wrapper to an ~uamqp.message.Message object. """ PROP_SEQ_NUMBER = b"x-opt-sequence-number" @@ -501,13 +516,14 @@ class EventData(object): def __init__(self, body=None, batch=None, message=None): """ - Initialize EventData + Initialize EventData. + :param body: The data to send in a single message. :type body: str, bytes or list :param batch: A data generator to send batched messages. :type batch: Generator :param message: The received message. - :type message: ~uamqp.Message + :type message: ~uamqp.message.Message """ self._partition_key = types.AMQPSymbol(EventData.PROP_PARTITION_KEY) self._annotations = {} @@ -536,7 +552,8 @@ def __init__(self, body=None, batch=None, message=None): def sequence_number(self): """ The sequence number of the event data object. - :returns: int + + :rtype: int """ return self._annotations.get(EventData.PROP_SEQ_NUMBER, None) @@ -544,7 +561,8 @@ def sequence_number(self): def offset(self): """ The offset of the event data object. - :returns: int + + :rtype: int """ try: return self._annotations[EventData.PROP_OFFSET].decode('UTF-8') @@ -555,7 +573,8 @@ def offset(self): def enqueued_time(self): """ The enqueued timestamp of the event data object. - :returns: datetime.datetime + + :rtype: datetime.datetime """ timestamp = self._annotations.get(EventData.PROP_TIMESTAMP, None) if timestamp: @@ -567,7 +586,8 @@ def device_id(self): """ The device ID of the event data object. This is only used for IoT Hub implementations. - :returns: bytes + + :rtype: bytes """ return self._annotations.get(EventData.PROP_DEVICE_ID, None) @@ -575,7 +595,8 @@ def device_id(self): def partition_key(self): """ The partition key of the event data object. - :returns: bytes + + :rtype: bytes """ try: return self._annotations[self._partition_key] @@ -586,6 +607,7 @@ def partition_key(self): def partition_key(self, value): """ Set the partition key of the event data object. + :param value: The partition key to set. :type value: str or bytes """ @@ -599,7 +621,8 @@ def partition_key(self, value): def properties(self): """ Application defined properties on the message. - :returns: dict + + :rtype: dict """ return self._properties @@ -607,6 +630,7 @@ def properties(self): def properties(self, value): """ Application defined properties on the message. + :param value: The application properties for the EventData. :type value: dict """ @@ -618,7 +642,8 @@ def properties(self, value): def body(self): """ The body of the event data object. - :returns: bytes or generator[bytes] + + :rtype: bytes or generator[bytes] """ return self.message.get_data() @@ -643,6 +668,7 @@ class Offset(object): def __init__(self, value, inclusive=False): """ Initialize Offset. + :param value: The offset value. :type value: ~datetime.datetime or int or str :param inclusive: Whether to include the supplied value as the start point. @@ -654,7 +680,8 @@ def __init__(self, value, inclusive=False): def selector(self): """ Creates a selector expression of the offset. - :returns: bytes + + :rtype: bytes """ operator = ">=" if self.inclusive else ">" if isinstance(self.value, datetime.datetime): diff --git a/azure/eventhub/async/__init__.py b/azure/eventhub/async/__init__.py index ba75500..e2e80d5 100644 --- a/azure/eventhub/async/__init__.py +++ b/azure/eventhub/async/__init__.py @@ -34,7 +34,7 @@ class EventHubClientAsync(EventHubClient): def _create_auth(self, auth_uri, username, password): # pylint: disable=no-self-use """ - Create an ~uamqp.authentication.SASTokenAuthAsync instance to authenticate + Create an ~uamqp.async.authentication_async.SASTokenAuthAsync instance to authenticate the session. :param auth_uri: The URI to authenticate against. @@ -48,7 +48,7 @@ def _create_auth(self, auth_uri, username, password): # pylint: disable=no-self def _create_connection_async(self): """ - Create a new ~uamqp.ConnectionAsync instance that will be shared between all + Create a new ~uamqp.async.connection_async.ConnectionAsync instance that will be shared between all AsyncSender/AsyncReceiver clients. """ if not self.connection: @@ -81,7 +81,7 @@ async def run_async(self): Run the EventHubClient asynchronously. Opens the connection and starts running all AsyncSender/AsyncReceiver clients. - :returns: ~azure.eventhub.EventHubClientAsync + :rtype: ~azure.eventhub.async.EventHubClientAsync """ log.info("{}: Starting {} clients".format(self.container_id, len(self.clients))) self._create_connection_async() @@ -101,7 +101,8 @@ async def stop_async(self): async def get_eventhub_info_async(self): """ Get details on the specified EventHub async. - :returns: dict + + :rtype: dict """ eh_name = self.address.path.lstrip('/') target = "amqps://{}/{}".format(self.address.hostname, eh_name) @@ -126,6 +127,7 @@ async def get_eventhub_info_async(self): def add_async_receiver(self, consumer_group, partition, offset=None, prefetch=300, loop=None): """ Add an async receiver to the client for a particular consumer group and partition. + :param consumer_group: The name of the consumer group. :type consumer_group: str :param partition: The ID of the partition. @@ -134,7 +136,7 @@ def add_async_receiver(self, consumer_group, partition, offset=None, prefetch=30 :type offset: ~azure.eventhub.Offset :param prefetch: The message prefetch count of the receiver. Default is 300. :type prefetch: int - :returns: ~azure.eventhub.ReceiverAsync + :rtype: ~azure.eventhub.async.ReceiverAsync """ source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( self.address.hostname, self.address.path, consumer_group, partition) @@ -151,6 +153,7 @@ def add_async_epoch_receiver(self, consumer_group, partition, epoch, prefetch=30 can connect to a partition at any given time - additional epoch receivers must have a higher epoch value or they will be rejected. If a 2nd epoch receiver has connected, the first will be closed. + :param consumer_group: The name of the consumer group. :type consumer_group: str :param partition: The ID of the partition. @@ -159,7 +162,7 @@ def add_async_epoch_receiver(self, consumer_group, partition, epoch, prefetch=30 :type epoch: int :param prefetch: The message prefetch count of the receiver. Default is 300. :type prefetch: int - :returns: ~azure.eventhub.ReceiverAsync + :rtype: ~azure.eventhub.async.ReceiverAsync """ source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( self.address.hostname, self.address.path, consumer_group, partition) @@ -171,11 +174,12 @@ def add_async_sender(self, partition=None, loop=None): """ Add an async sender to the client to send ~azure.eventhub.EventData object to an EventHub. + :param partition: Optionally specify a particular partition to send to. If omitted, the events will be distributed to available partitions via - round-robin + round-robin. :type partition: str - :returns: ~azure.eventhub.SenderAsync + :rtype: ~azure.eventhub.async.SenderAsync """ target = "amqps://{}{}".format(self.address.hostname, self.address.path) handler = AsyncSender(self, target, partition=partition, loop=loop) @@ -190,8 +194,9 @@ class AsyncSender(Sender): def __init__(self, client, target, partition=None, loop=None): # pylint: disable=super-init-not-called """ Instantiate an EventHub event SenderAsync client. - :param client: The parent EventHubClient. - :type client: ~azure.eventhub.EventHubClient. + + :param client: The parent EventHubClientAsync. + :type client: ~azure.eventhub.async.EventHubClientAsync :param target: The URI of the EventHub to send to. :type target: str :param loop: An event loop. @@ -213,6 +218,7 @@ async def send(self, event_data): """ Sends an event data and asynchronously waits until acknowledgement is received or operation times out. + :param event_data: The event to be sent. :type event_data: ~azure.eventhub.EventData :raises: ~azure.eventhub.EventHubError if the message fails to @@ -237,10 +243,11 @@ class AsyncReceiver(Receiver): def __init__(self, client, source, prefetch=300, epoch=None, loop=None): # pylint: disable=super-init-not-called """ Instantiate an async receiver. - :param client: The parent EventHubClient. - :type client: ~azure.eventhub.EventHubClient + + :param client: The parent EventHubClientAsync. + :type client: ~azure.eventhub.async.EventHubClientAsync :param source: The source EventHub from which to receive events. - :type source: ~uamqp.Source + :type source: ~uamqp.address.Source :param prefetch: The number of events to prefetch from the service for processing. Default is 300. :type prefetch: int @@ -268,6 +275,7 @@ def __init__(self, client, source, prefetch=300, epoch=None, loop=None): # pyli async def receive(self, max_batch_size=None, callback=None, timeout=None): """ Receive events asynchronously from the EventHub. + :param max_batch_size: Receive a batch of events. Batch size will be up to the maximum specified, but will return as soon as service returns no new events. If combined with a timeout and no events are @@ -278,7 +286,7 @@ async def receive(self, max_batch_size=None, callback=None, timeout=None): be a function that accepts a single argument - the event data. This callback will be run before the message is returned in the result generator. :type callback: func[~azure.eventhub.EventData] - :returns: list[~azure.eventhub.EventData] + :rtype: list[~azure.eventhub.EventData] """ try: self._callback = callback diff --git a/azure/eventprocessorhost/abstract_checkpoint_manager.py b/azure/eventprocessorhost/abstract_checkpoint_manager.py index 081ebd4..5e6ec84 100644 --- a/azure/eventprocessorhost/abstract_checkpoint_manager.py +++ b/azure/eventprocessorhost/abstract_checkpoint_manager.py @@ -20,8 +20,10 @@ def __init__(self): async def create_checkpoint_store_if_not_exists_async(self): """ Create the checkpoint store if it doesn't exist. Do nothing if it does exist. - :returns: `True` if the checkpoint store already exists or was created OK, `False` + + :return: `True` if the checkpoint store already exists or was created OK, `False` if there was a failure. + :rtype: bool """ pass @@ -30,9 +32,11 @@ async def get_checkpoint_async(self, partition_id): """ Get the checkpoint data associated with the given partition. Could return null if no checkpoint has been created for that partition. + :param partition_id: The ID of a given parition. :type partition_id: str - :returns: Given partition checkpoint info, or `None` if none has been previously stored. + :return: Given partition checkpoint info, or `None` if none has been previously stored. + :rtype: ~azure.eventprocessorhost.checkpoint.Checkpoint """ pass @@ -41,20 +45,23 @@ async def create_checkpoint_if_not_exists_async(self, partition_id): """ Create the given partition checkpoint if it doesn't exist.Do nothing if it does exist. The offset/sequenceNumber for a freshly-created checkpoint should be set to StartOfStream/0. + :param partition_id: The ID of a given parition. :type partition_id: str - :returns: The checkpoint for the given partition, whether newly created or already existing. + :return: The checkpoint for the given partition, whether newly created or already existing. + :rtype: ~azure.eventprocessorhost.checkpoint.Checkpoint """ pass @abstractmethod async def update_checkpoint_async(self, lease, checkpoint): """ - Update the checkpoint in the store with the offset/sequenceNumber in the provided checkpoint + Update the checkpoint in the store with the offset/sequenceNumber in the provided checkpoint. + :param lease: The lease to be updated. - :type lease: ~azure.eventprocessorhost.Lease + :type lease: ~azure.eventprocessorhost.lease.Lease :param checkpoint: offset/sequeceNumber to update the store with. - :type checkpoint: ~azure.eventprocessorhost.Checkpoint + :type checkpoint: ~azure.eventprocessorhost.checkpoint.Checkpoint """ pass @@ -63,6 +70,7 @@ async def delete_checkpoint_async(self, partition_id): """ Delete the stored checkpoint for the given partition. If there is no stored checkpoint for the given partition, that is treated as success. + :param partition_id: The ID of a given parition. :type partition_id: str """ diff --git a/azure/eventprocessorhost/abstract_event_processor.py b/azure/eventprocessorhost/abstract_event_processor.py index fa4020b..6ff9dd4 100644 --- a/azure/eventprocessorhost/abstract_event_processor.py +++ b/azure/eventprocessorhost/abstract_event_processor.py @@ -16,8 +16,9 @@ def __init__(self, params=None): async def open_async(self, context): """ Called by processor host to initialize the event processor. + :param context: Information about the partition - :type context: ~azure.eventprocessorhost.PartitionContext + :type context: ~azure.eventprocessorhost.partition_context.PartitionContext """ pass @@ -25,8 +26,9 @@ async def open_async(self, context): async def close_async(self, context, reason): """ Called by processor host to indicate that the event processor is being stopped. + :param context: Information about the partition - :type context: ~azure.eventprocessorhost.PartitionContext + :type context: ~azure.eventprocessorhost.partition_context.PartitionContext :param reason: The reason for closing. :type reason: str """ @@ -37,10 +39,11 @@ async def process_events_async(self, context, messages): """ Called by the processor host when a batch of events has arrived. This is where the real work of the event processor is done. + :param context: Information about the partition - :type context: ~azure.eventprocessorhost.PartitionContext + :type context: ~azure.eventprocessorhost.partition_context.PartitionContext :param messages: The events to be processed. - :type messages: list of ~azure.eventhub.EventData + :type messages: list[~azure.eventhub.EventData] """ pass @@ -49,9 +52,10 @@ async def process_error_async(self, context, error): """ Called when the underlying client experiences an error while receiving. EventProcessorHost will take care of recovering from the error and - continuing to pump messages,so no action is required from + continuing to pump messages. + :param context: Information about the partition - :type context: ~azure.eventprocessorhost.PartitionContext + :type context: ~azure.eventprocessorhost.partition_context.PartitionContext :param error: The error that occured. """ pass diff --git a/azure/eventprocessorhost/abstract_lease_manager.py b/azure/eventprocessorhost/abstract_lease_manager.py index f9102e6..8638e49 100644 --- a/azure/eventprocessorhost/abstract_lease_manager.py +++ b/azure/eventprocessorhost/abstract_lease_manager.py @@ -25,7 +25,9 @@ def __init__(self, lease_renew_interval, lease_duration): async def create_lease_store_if_not_exists_async(self): """ Create the lease store if it does not exist, do nothing if it does exist. - :returns: `True` if the lease store already exists or was created successfully, `False` if not. + + :return: `True` if the lease store already exists or was created successfully, `False` if not. + :rtype: bool """ pass @@ -33,7 +35,9 @@ async def create_lease_store_if_not_exists_async(self): async def delete_lease_store_async(self): """ Not used by EventProcessorHost, but a convenient function to have for testing. - :returns: `True` if the lease store was deleted successfully, `False` if not. + + :return: `True` if the lease store was deleted successfully, `False` if not. + :rtype: bool """ pass @@ -41,9 +45,11 @@ async def get_lease_async(self, partition_id): """ Return the lease info for the specified partition. Can return null if no lease has been created in the store for the specified partition. + :param partition_id: The ID of a given partition. :type parition_id: str - :returns: lease info for the partition, or `None`. + :return: Lease info for the partition, or `None`. + :rtype: """ pass @@ -52,7 +58,9 @@ def get_all_leases(self): """ Return the lease info for all partitions. A typical implementation could just call get_lease_async() on all partitions. - :returns: list of lease info. + + :return: A list of lease info. + :rtype: """ pass @@ -61,9 +69,10 @@ async def create_lease_if_not_exists_async(self, partition_id): """ Create in the store the lease info for the given partition, if it does not exist. Do nothing if it does exist in the store already. + :param partition_id: The ID of a given partition. :type parition_id: str - :returns: The existing or newly-created lease info for the partition. + :return: The existing or newly-created lease info for the partition. """ pass @@ -72,8 +81,9 @@ async def delete_lease_async(self, lease): """ Delete the lease info for the given partition from the store. If there is no stored lease for the given partition, that is treated as success. + :param lease: The lease to be deleted. - :type lease: ~azure.eventprocessorhost.Lease + :type lease: ~azure.eventprocessorhost.lease.Lease """ pass @@ -83,9 +93,11 @@ async def acquire_lease_async(self, lease): Acquire the lease on the desired partition for this EventProcessorHost. Note that it is legal to acquire a lease that is already owned by another host. Lease-stealing is how partitions are redistributed when additional hosts are started. + :param lease: The lease to be acquired. - :type lease: ~azure.eventprocessorhost.Lease - :returns: `True` if the lease was acquired successfully, `False` if not. + :type lease: ~azure.eventprocessorhost.lease.Lease + :return: `True` if the lease was acquired successfully, `False` if not. + :rtype: bool """ pass @@ -95,9 +107,11 @@ async def renew_lease_async(self, lease): Renew a lease currently held by this host. If the lease has been stolen, or expired, or released, it is not possible to renew it. You will have to call get_lease_async() and then acquire_lease_async() again. + :param lease: The lease to be renewed. - :type lease: ~azure.eventprocessorhost.Lease - :returns: `True` if the lease was renewed successfully, `False` if not. + :type lease: ~azure.eventprocessorhost.lease.Lease + :return: `True` if the lease was renewed successfully, `False` if not. + :rtype: bool """ pass @@ -106,9 +120,11 @@ async def release_lease_async(self, lease): """ Give up a lease currently held by this host. If the lease has been stolen, or expired, releasing it is unnecessary, and will fail if attempted. + :param lease: The lease to be released. - :type lease: ~azure.eventprocessorhost.Lease - :returns: `True` if the lease was released successfully, `False` if not. + :type lease: ~azure.eventprocessorhost.lease.Lease + :return: `True` if the lease was released successfully, `False` if not. + :rtype: bool """ pass @@ -119,9 +135,10 @@ async def update_lease_async(self, lease): hold a lease in order to update it. If the lease has been stolen, or expired, or released, it cannot be updated. Updating should renew the lease before performing the update to avoid lease expiration during the process. + :param lease: The lease to be updated. - :type lease: ~azure.eventprocessorhost.Lease - :returns: `True` if the updated was performed successfully, `False` if not. + :type lease: ~azure.eventprocessorhost.lease.Lease + :return: `True` if the updated was performed successfully, `False` if not. + :rtype: bool """ pass - \ No newline at end of file diff --git a/azure/eventprocessorhost/azure_blob_lease.py b/azure/eventprocessorhost/azure_blob_lease.py index 6c01760..04ec135 100644 --- a/azure/eventprocessorhost/azure_blob_lease.py +++ b/azure/eventprocessorhost/azure_blob_lease.py @@ -16,7 +16,7 @@ class AzureBlobLease(Lease): def __init__(self): """ - Init Azure Blob Lease + Init Azure Blob Lease. """ super() Lease.__init__(self) @@ -25,7 +25,7 @@ def __init__(self): def serializable(self): """ - Returns Serialiazble instance of __dict__ + Returns Serialiazble instance of `__dict__`. """ serial = self.__dict__.copy() del serial['state'] @@ -33,13 +33,13 @@ def serializable(self): def with_lease(self, lease): """ - Init with exisiting lease + Init with exisiting lease. """ super().with_source(lease) def with_blob(self, blob): """ - Init Azure Blob Lease with existing blob + Init Azure Blob Lease with existing blob. """ content = json.loads(blob.content) self.partition_id = content["partition_id"] @@ -51,7 +51,7 @@ def with_blob(self, blob): def with_source(self, lease): """ - Init Azure Blob Lease from existing + Init Azure Blob Lease from existing. """ super().with_source(lease) self.offset = lease.offset @@ -59,7 +59,7 @@ def with_source(self, lease): async def is_expired(self): """ - Check and return azure blob lease state using storage api + Check and return Azure Blob Lease state using Storage API. """ if asyncio.iscoroutinefunction(self.state): current_state = await self.state() diff --git a/azure/eventprocessorhost/azure_storage_checkpoint_manager.py b/azure/eventprocessorhost/azure_storage_checkpoint_manager.py index 261ea43..a631fdb 100644 --- a/azure/eventprocessorhost/azure_storage_checkpoint_manager.py +++ b/azure/eventprocessorhost/azure_storage_checkpoint_manager.py @@ -76,8 +76,10 @@ def initialize(self, host): async def create_checkpoint_store_if_not_exists_async(self): """ Create the checkpoint store if it doesn't exist. Do nothing if it does exist. - :returns: `True` if the checkpoint store already exists or was created OK, `False` - if there was a failure + + :return: `True` if the checkpoint store already exists or was created OK, `False` + if there was a failure. + :rtype: bool """ await self.create_lease_store_if_not_exists_async() @@ -85,7 +87,11 @@ async def get_checkpoint_async(self, partition_id): """ Get the checkpoint data associated with the given partition. Could return null if no checkpoint has been created for that partition. - :returns: Given partition checkpoint info, or `None` if none has been previously stored. + + :param partition_id: The partition ID. + :type partition_id: str + :return: Given partition checkpoint info, or `None` if none has been previously stored. + :rtype: ~azure.eventprocessorhost.checkpoint.Checkpoint """ lease = await self.get_lease_async(partition_id) checkpoint = None @@ -99,7 +105,11 @@ async def create_checkpoint_if_not_exists_async(self, partition_id): """ Create the given partition checkpoint if it doesn't exist.Do nothing if it does exist. The offset/sequenceNumber for a freshly-created checkpoint should be set to StartOfStream/0. - :returns: The checkpoint for the given partition, whether newly created or already existing. + + :param partition_id: The partition ID. + :type partition_id: str + :return: The checkpoint for the given partition, whether newly created or already existing. + :rtype: ~azure.eventprocessorhost.checkpoint.Checkpoint """ checkpoint = await self.get_checkpoint_async(partition_id) if not checkpoint: @@ -111,6 +121,11 @@ async def update_checkpoint_async(self, lease, checkpoint): """ Update the checkpoint in the store with the offset/sequenceNumber in the provided checkpoint checkpoint:offset/sequeceNumber to update the store with. + + :param lease: The stored lease to be updated. + :type lease: ~azure.eventprocessorhost.lease.Lease + :param checkpoint: The checkpoint to update the lease with. + :type checkpoint: ~azure.eventprocessorhost.checkpoint.Checkpoint """ new_lease = AzureBlobLease() new_lease.with_source(lease) @@ -122,6 +137,9 @@ async def delete_checkpoint_async(self, partition_id): """ Delete the stored checkpoint for the given partition. If there is no stored checkpoint for the given partition, that is treated as success. + + :param partition_id: The partition ID. + :type partition_id: str """ return # Make this a no-op to avoid deleting leases by accident. @@ -130,7 +148,9 @@ async def delete_checkpoint_async(self, partition_id): async def create_lease_store_if_not_exists_async(self): """ Create the lease store if it does not exist, do nothing if it does exist. - :returns: `True` if the lease store already exists or was created successfully, `False` if not. + + :return: `True` if the lease store already exists or was created successfully, `False` if not. + :rtype: bool """ try: await self.host.loop.run_in_executor( @@ -148,7 +168,9 @@ async def create_lease_store_if_not_exists_async(self): async def delete_lease_store_async(self): """ Not used by EventProcessorHost, but a convenient function to have for testing. - :returns: `True` if the lease store was deleted successfully, `False` if not. + + :return: `True` if the lease store was deleted successfully, `False` if not. + :rtype: bool """ return "Not Supported in Python" @@ -156,7 +178,11 @@ async def get_lease_async(self, partition_id): """ Return the lease info for the specified partition. Can return null if no lease has been created in the store for the specified partition. - :returns: lease info for the partition, or `None`. + + :param partition_id: The partition ID. + :type partition_id: str + :return: lease info for the partition, or `None`. + :rtype: ~azure.eventprocessorhost.lease.Lease """ try: blob = await self.host.loop.run_in_executor( @@ -191,7 +217,9 @@ async def get_all_leases(self): """ Return the lease info for all partitions. A typical implementation could just call get_lease_async() on all partitions. - :returns: list of lease info. + + :return: A list of lease info. + :rtype: list[~azure.eventprocessorhost.lease.Lease] """ lease_futures = [] partition_ids = await self.host.partition_manager.get_partition_ids_async() @@ -203,9 +231,11 @@ async def create_lease_if_not_exists_async(self, partition_id): """ Create in the store the lease info for the given partition, if it does not exist. Do nothing if it does exist in the store already. + :param partition_id: The ID of a given parition. :type partition_id: str - :returns: the existing or newly-created lease info for the partition. + :return: the existing or newly-created lease info for the partition. + :rtype: ~azure.eventprocessorhost.lease.Lease """ return_lease = None try: @@ -235,6 +265,9 @@ async def delete_lease_async(self, lease): """ Delete the lease info for the given partition from the store. If there is no stored lease for the given partition, that is treated as success. + + :param lease: The stored lease to be deleted. + :type lease: ~azure.eventprocessorhost.lease.Lease """ await self.host.loop.run_in_executor( self.executor, @@ -249,7 +282,11 @@ async def acquire_lease_async(self, lease): Acquire the lease on the desired partition for this EventProcessorHost. Note that it is legal to acquire a lease that is already owned by another host. Lease-stealing is how partitions are redistributed when additional hosts are started. - :returns: `True` if the lease was acquired successfully, `False` if not. + + :param lease: The stored lease to be acquired. + :type lease: ~azure.eventprocessorhost.lease.Lease + :return: `True` if the lease was acquired successfully, `False` if not. + :rtype: bool """ retval = True new_lease_id = str(uuid.uuid4()) @@ -307,7 +344,11 @@ async def renew_lease_async(self, lease): Renew a lease currently held by this host. If the lease has been stolen, or expired, or released, it is not possible to renew it. You will have to call getLease() and then acquireLease() again. - :returns: `True` if the lease was renewed successfully, `False` if not. + + :param lease: The stored lease to be renewed. + :type lease: ~azure.eventprocessorhost.lease.Lease + :return: `True` if the lease was renewed successfully, `False` if not. + :rtype: bool """ try: await self.host.loop.run_in_executor( @@ -331,7 +372,11 @@ async def release_lease_async(self, lease): """ Give up a lease currently held by this host. If the lease has been stolen, or expired, releasing it is unnecessary, and will fail if attempted. - :returns: `True` if the lease was released successfully, `False` if not. + + :param lease: The stored lease to be released. + :type lease: ~azure.eventprocessorhost.lease.Lease + :return: `True` if the lease was released successfully, `False` if not. + :rtype: bool """ lease_id = None try: @@ -369,7 +414,11 @@ async def update_lease_async(self, lease): hold a lease in order to update it. If the lease has been stolen, or expired, or released, it cannot be updated. Updating should renew the lease before performing the update to avoid lease expiration during the process. - :returns: `True` if the updated was performed successfully, `False` if not. + + :param lease: The stored lease to be updated. + :type lease: ~azure.eventprocessorhost.lease.Lease + :return: `True` if the updated was performed successfully, `False` if not. + :rtype: bool """ if lease is None: return False diff --git a/azure/eventprocessorhost/cancellation_token.py b/azure/eventprocessorhost/cancellation_token.py index 0764af6..ae1aeae 100644 --- a/azure/eventprocessorhost/cancellation_token.py +++ b/azure/eventprocessorhost/cancellation_token.py @@ -8,13 +8,13 @@ """ class CancellationToken: """ - Thread Safe Mutable Cancellation Token + Thread Safe Mutable Cancellation Token. """ def __init__(self): self.is_cancelled = False def cancel(self): """ - Cancel the token + Cancel the token. """ self.is_cancelled = True diff --git a/azure/eventprocessorhost/checkpoint.py b/azure/eventprocessorhost/checkpoint.py index 48b03f1..ff09052 100644 --- a/azure/eventprocessorhost/checkpoint.py +++ b/azure/eventprocessorhost/checkpoint.py @@ -5,11 +5,12 @@ class Checkpoint: """ - Contains checkpoint metadata + Contains checkpoint metadata. """ def __init__(self, partition_id, offset="-1", sequence_number="0"): - """Initialize Checkpoint + """Initialize Checkpoint. + :param partition_id: The parition ID of the checkpoint. :type partition_id: str :param offset: The receive offset of the checkpoint. @@ -24,8 +25,9 @@ def __init__(self, partition_id, offset="-1", sequence_number="0"): def from_source(self, checkpoint): """ Creates a new Checkpoint from an existing checkpoint. + :param checkpoint: Existing checkpoint. - :type checkpoint: ~azure.eventprocessorhost.Checkpoint + :type checkpoint: ~azure.eventprocessorhost.checkpoint.Checkpoint """ self.partition_id = checkpoint.partition_id self.offset = checkpoint.offset diff --git a/azure/eventprocessorhost/eh_config.py b/azure/eventprocessorhost/eh_config.py index b977416..73f89a8 100644 --- a/azure/eventprocessorhost/eh_config.py +++ b/azure/eventprocessorhost/eh_config.py @@ -12,6 +12,7 @@ class EventHubConfig: """ A container class for Event Hub properties. + :param sb_name: The EventHub (ServiceBus) namespace. :type sb_name: str :param eh_name: The EventHub name. @@ -43,6 +44,8 @@ def get_client_address(self): """ Returns an auth token dictionary for making calls to eventhub REST API. + + :rtype: str """ return "amqps://{}:{}@{}.{}:5671/{}".format( urllib.parse.quote_plus(self.policy), @@ -54,6 +57,8 @@ def get_client_address(self): def get_rest_token(self): """ Returns an auth token for making calls to eventhub REST API. + + :rtype: str """ uri = urllib.parse.quote_plus( "https://{}.{}/{}".format(self.sb_name, self.namespace_suffix, self.eh_name)) diff --git a/azure/eventprocessorhost/eh_partition_pump.py b/azure/eventprocessorhost/eh_partition_pump.py index 87db09d..738422d 100644 --- a/azure/eventprocessorhost/eh_partition_pump.py +++ b/azure/eventprocessorhost/eh_partition_pump.py @@ -15,7 +15,7 @@ class EventHubPartitionPump(PartitionPump): """ - Pulls and messages from lease partition from eventhub and sends them to processor + Pulls and messages from lease partition from eventhub and sends them to processor. """ def __init__(self, host, lease): @@ -27,7 +27,7 @@ def __init__(self, host, lease): async def on_open_async(self): """ - Eventhub Override for on_open_async + Eventhub Override for on_open_async. """ _opened_ok = False _retry_count = 0 @@ -61,7 +61,7 @@ async def on_open_async(self): async def open_clients_async(self): """ Responsible for establishing connection to event hub client - throws EventHubsException, IOException, InterruptedException, ExecutionException + throws EventHubsException, IOException, InterruptedException, ExecutionException. """ await self.partition_context.get_initial_offset_async() # Create event hub client and receive handler and set options @@ -78,7 +78,7 @@ async def open_clients_async(self): async def clean_up_clients_async(self): """ - Resets the pump swallows all exceptions + Resets the pump swallows all exceptions. """ if self.partition_receiver: if self.eh_client: @@ -89,7 +89,8 @@ async def clean_up_clients_async(self): async def on_closing_async(self, reason): """ - Overides partition pump on cleasing + Overides partition pump on cleasing. + :param reason: The reason for the shutdown. :type reason: str """ @@ -100,7 +101,7 @@ async def on_closing_async(self, reason): class PartitionReceiver: """ - Recieves events from a async until lease is lost + Recieves events asynchronously until lease is lost. """ def __init__(self, eh_partition_pump): @@ -110,7 +111,7 @@ def __init__(self, eh_partition_pump): async def run(self): """ - Runs the async partion reciever event loop to retrive messages from the event queue + Runs the async partion reciever event loop to retrive messages from the event queue. """ # Implement pull max batch from queue instead of one message at a time while self.eh_partition_pump.pump_status != "Errored" and not self.eh_partition_pump.is_closing(): @@ -135,10 +136,11 @@ async def process_events_async(self, events): """ This method is called on the thread that the EH client uses to run the pump. There is one pump per EventHubClient. Since each PartitionPump creates a - new EventHubClient,using that thread to call OnEvents does no harm. Even if OnEvents + new EventHubClient, using that thread to call OnEvents does no harm. Even if OnEvents is slow, the pump will get control back each time OnEvents returns, and be able to receive a new batch of messages with which to make the next OnEvents call.The pump gains nothing by running faster than OnEvents. + :param events: List of events to be processed. :type events: list of ~azure.eventhub.EventData """ @@ -147,7 +149,8 @@ async def process_events_async(self, events): async def process_error_async(self, error): """ Handles processing errors this is never called since python recieve client doesn't - have error handling implemented (TBD add fault pump handling) + have error handling implemented (TBD add fault pump handling). + :param error: An error the occurred. :type error: Exception """ diff --git a/azure/eventprocessorhost/eph.py b/azure/eventprocessorhost/eph.py index 778a047..c344628 100644 --- a/azure/eventprocessorhost/eph.py +++ b/azure/eventprocessorhost/eph.py @@ -11,25 +11,26 @@ class EventProcessorHost: """ Represents a host for processing Event Hubs event data at scale. - Takes in event hub a event processor class definition a eh_config object - As well as a storage manager and an optional event_processor params (ep_params) + Takes in an event hub, a event processor class definition, a config object, + as well as a storage manager and optional event processor params (ep_params). """ def __init__(self, event_processor, eh_config, storage_manager, ep_params=None, eph_options=None, loop=None): """ Initialize EventProcessorHost. + :param event_processor: The event processing handler. - :type event_processor: ~azure.eventprocessorhost.AbstractEventProcessor + :type event_processor: ~azure.eventprocessorhost.abstract_event_processor.AbstractEventProcessor :param eh_config: The EPH connection configuration. - :type eh_config: ~azure.eventprocessorhost.EventHubConfig + :type eh_config: ~azure.eventprocessorhost.eh_config.EventHubConfig :param storage_manager: The Azure storage manager for persisting lease and checkpoint information. - :type storage_manager: ~azure.eventprocessorhost.AzureStorageCheckpointLeaseManager + :type storage_manager: ~azure.eventprocessorhost.azure_storage_checkpoint_manager.AzureStorageCheckpointLeaseManager :param ep_params: Optional arbitrary parameters to be passed into the event_processor on initialization. :type ep_params: list :param eph_options: EPH configuration options. - :type eph_options: ~azure.eventprocessorhost.EPHOptions + :type eph_options: ~azure.eventprocessorhost.eph.EPHOptions :param loop: An eventloop. If not provided the default asyncio event loop will be used. """ self.event_processor = event_processor @@ -46,7 +47,7 @@ def __init__(self, event_processor, eh_config, storage_manager, ep_params=None, async def open_async(self): """ - Starts the host + Starts the host. """ if not self.loop: self.loop = asyncio.get_event_loop() @@ -54,14 +55,14 @@ async def open_async(self): async def close_async(self): """ - Stops the host + Stops the host. """ await self.partition_manager.stop_async() class EPHOptions: """ - Class that contains default and overidable EPH option + Class that contains default and overidable EPH option. """ def __init__(self): diff --git a/azure/eventprocessorhost/lease.py b/azure/eventprocessorhost/lease.py index 6c45371..6d4e0b1 100644 --- a/azure/eventprocessorhost/lease.py +++ b/azure/eventprocessorhost/lease.py @@ -18,7 +18,8 @@ def __init__(self): def with_partition_id(self, partition_id): """ - Init with partition Id + Init with partition Id. + :param partition_id: ID of a given partition. :type partition_id: str """ @@ -30,8 +31,9 @@ def with_partition_id(self, partition_id): def with_source(self, lease): """ Init with existing lease. + :param lease: An existing Lease. - :type lease: ~azure.eventprocessorhost.Lease + :type lease: ~azure.eventprocessorhost.lease.Lease """ self.partition_id = lease.partition_id self.epoch = lease.epoch @@ -42,13 +44,14 @@ async def is_expired(self): """ Determines whether the lease is expired. By default lease never expires. Deriving class implements the lease expiry logic. - :returns: bool + + :rtype: bool """ return False def increment_epoch(self): """ - Increment lease epoch + Increment lease epoch. """ self.epoch += 1 return self.epoch diff --git a/azure/eventprocessorhost/partition_context.py b/azure/eventprocessorhost/partition_context.py index d4cfa19..fb619b3 100644 --- a/azure/eventprocessorhost/partition_context.py +++ b/azure/eventprocessorhost/partition_context.py @@ -12,7 +12,7 @@ class PartitionContext: """ - Encapsulates information related to an Event Hubs partition used by AbstractEventProcessor + Encapsulates information related to an Event Hubs partition used by AbstractEventProcessor. """ def __init__(self, host, partition_id, eh_path, consumer_group_name, pump_loop=None): @@ -28,6 +28,7 @@ def __init__(self, host, partition_id, eh_path, consumer_group_name, pump_loop=N def set_offset_and_sequence_number(self, event_data): """ Updates offset based on event. + :param event_data: A received EventData with valid offset and sequenceNumber. :type event_data: ~azure.eventhub.EventData """ @@ -39,7 +40,8 @@ def set_offset_and_sequence_number(self, event_data): async def get_initial_offset_async(self): # throws InterruptedException, ExecutionException """ Gets the initial offset for processing the partition. - :returns: str + + :rtype: str """ _logger.info("Calling user-provided initial offset provider {} {}".format( self.host.guid, self.partition_id)) @@ -60,7 +62,7 @@ async def get_initial_offset_async(self): # throws InterruptedException, Executi async def checkpoint_async(self): """ Generates a checkpoint for the partition using the curren offset and sequenceNumber for - and persists to the checkpoint manager + and persists to the checkpoint manager. """ captured_checkpoint = Checkpoint(self.partition_id, self.offset, self.sequence_number) await self.persist_checkpoint_async(captured_checkpoint) @@ -69,9 +71,10 @@ async def checkpoint_async_event_data(self, event_data): """ Stores the offset and sequenceNumber from the provided received EventData instance, then writes those values to the checkpoint store via the checkpoint manager. + :param event_data: A received EventData with valid offset and sequenceNumber. :type event_data: ~azure.eventhub.EventData - :raises: ValueError if suplied event_data is None + :raises: ValueError if suplied event_data is None. :raises: ValueError if the sequenceNumber is less than the last checkpointed value. """ if not event_data: @@ -89,7 +92,7 @@ def to_string(self): Returns the parition context in the following format: "PartitionContext({EventHubPath}{ConsumerGroupName}{PartitionId}{SequenceNumber})" - :returns: str + :rtype: str """ return "PartitionContext({}{}{}{})".format(self.eh_path, self.consumer_group_name, diff --git a/azure/eventprocessorhost/partition_manager.py b/azure/eventprocessorhost/partition_manager.py index 8c1e53a..2580abc 100644 --- a/azure/eventprocessorhost/partition_manager.py +++ b/azure/eventprocessorhost/partition_manager.py @@ -18,7 +18,7 @@ class PartitionManager: """ - Manages the partition event pump execution + Manages the partition event pump execution. """ def __init__(self, host): @@ -30,7 +30,9 @@ def __init__(self, host): async def get_partition_ids_async(self): """ - Returns a list of all the event hub partition ids + Returns a list of all the event hub partition IDs. + + :rtype: list[str] """ if not self.partition_ids: eh_client = EventHubClientAsync( @@ -65,7 +67,7 @@ async def stop_async(self): async def run_async(self): """ - Starts the run loop and manages exceptions and cleanup + Starts the run loop and manages exceptions and cleanup. """ try: await self.run_loop_async() @@ -82,7 +84,10 @@ async def initialize_stores_async(self): """ Intializes the partition checkpoint and lease store ensures that a checkpoint exists for all partitions. Note in this case checkpoint and lease stores are - the same storage manager construct. Returns the number of partitions + the same storage manager construct. + + :return: Returns the number of partitions. + :rtype: int """ await self.host.storage_manager.create_checkpoint_store_if_not_exists_async() partition_ids = await self.get_partition_ids_async() @@ -102,7 +107,7 @@ async def initialize_stores_async(self): def retry(self, func, partition_id, retry_message, final_failure_message, max_retries, host_id): """ - Make attempt_renew_lease async call sync + Make attempt_renew_lease async call sync. """ loop = asyncio.new_event_loop() loop.run_until_complete(self.retry_async(func, partition_id, retry_message, @@ -111,7 +116,7 @@ def retry(self, func, partition_id, retry_message, final_failure_message, max_re async def retry_async(self, func, partition_id, retry_message, final_failure_message, max_retries, host_id): """ - Throws if it runs out of retries. If it returns, action succeeded + Throws if it runs out of retries. If it returns, action succeeded. """ created_okay = False retry_count = 0 @@ -127,7 +132,7 @@ async def retry_async(self, func, partition_id, retry_message, async def run_loop_async(self): """ - This is the main execution loop for allocating and manging pumps + This is the main execution loop for allocating and manging pumps. """ while not self.cancellation_token.is_cancelled: lease_manager = self.host.storage_manager @@ -189,7 +194,12 @@ async def run_loop_async(self): async def check_and_add_pump_async(self, partition_id, lease): """ - Updates the lease on an exisiting pump + Updates the lease on an exisiting pump. + + :param partition_id: The partition ID. + :type partition_id: str + :param lease: The lease to be used. + :type lease: ~azure.eventprocessorhost.lease.Lease """ if partition_id in self.partition_pumps: # There already is a pump. Make sure the pump is working and replace the lease. @@ -208,7 +218,12 @@ async def check_and_add_pump_async(self, partition_id, lease): async def create_new_pump_async(self, partition_id, lease): """ - Create a new pump thread with a given lease + Create a new pump thread with a given lease. + + :param partition_id: The partition ID. + :type partition_id: str + :param lease: The lease to be used. + :type lease: ~azure.eventprocessorhost.lease.Lease """ loop = asyncio.get_event_loop() partition_pump = EventHubPartitionPump(self.host, lease) @@ -219,7 +234,12 @@ async def create_new_pump_async(self, partition_id, lease): async def remove_pump_async(self, partition_id, reason): """ - Stops a single partiton pump + Stops a single partiton pump. + + :param partition_id: The partition ID. + :type partition_id: str + :param reason: A reason for closing. + :type reason: str """ if partition_id in self.partition_pumps: captured_pump = self.partition_pumps[partition_id] @@ -239,7 +259,11 @@ async def remove_pump_async(self, partition_id, reason): async def remove_all_pumps_async(self, reason): """ Stops all partition pumps - (Note this might be wrong and need to await all tasks before returning done) + (Note this might be wrong and need to await all tasks before returning done). + + :param reason: A reason for closing. + :type reason: str + :rtype: bool """ pump_tasks = [self.remove_pump_async(p_id, reason) for p_id in self.partition_pumps] await asyncio.gather(*pump_tasks) @@ -267,6 +291,12 @@ def which_lease_to_steal(self, stealable_leases, have_lease_count): biggest and this host by two at a time. If the starting difference is two or greater, then the difference cannot end up below 0. This host may become tied for biggest, but it cannot become larger than the host that it is stealing from. + + :param stealable_leases: List of leases to determine which can be stolen. + :type stealable_leases: list[~azure.eventprocessorhost.lease.Lease] + :param have_lease_count: Lease count. + :type have_lease_count: int + :rtype: ~azure.eventprocessorhost.lease.Lease """ counts_by_owner = self.count_leases_by_owner(stealable_leases) biggest_owner = (sorted(counts_by_owner.items(), key=lambda kv: kv[1])).pop() @@ -278,14 +308,14 @@ def which_lease_to_steal(self, stealable_leases, have_lease_count): def count_leases_by_owner(self, leases): # pylint: disable=no-self-use """ - Returns a dictionary of leases by current owner + Returns a dictionary of leases by current owner. """ owners = [l.owner for l in leases] return dict(Counter(owners)) def attempt_renew_lease(self, lease_task, owned_by_others_q, lease_manager): """ - Make attempt_renew_lease async call sync + Make attempt_renew_lease async call sync. """ loop = asyncio.new_event_loop() loop.run_until_complete(self.attempt_renew_lease_async(lease_task, owned_by_others_q, lease_manager)) @@ -293,7 +323,7 @@ def attempt_renew_lease(self, lease_task, owned_by_others_q, lease_manager): async def attempt_renew_lease_async(self, lease_task, owned_by_others_q, lease_manager): """ Attempts to renew a potential lease if possible and - marks in the queue as none adds to adds to the queue + marks in the queue as none adds to adds to the queue. """ try: possible_lease = await lease_task diff --git a/azure/eventprocessorhost/partition_pump.py b/azure/eventprocessorhost/partition_pump.py index 1327463..26e62b0 100644 --- a/azure/eventprocessorhost/partition_pump.py +++ b/azure/eventprocessorhost/partition_pump.py @@ -14,7 +14,7 @@ class PartitionPump(): """ - Manages individual connection to a given partition + Manages individual connection to a given partition. """ def __init__(self, host, lease): @@ -27,21 +27,22 @@ def __init__(self, host, lease): def run(self): """ - Makes pump sync so that it can be run in a thread + Makes pump sync so that it can be run in a thread. """ self.loop = asyncio.new_event_loop() self.loop.run_until_complete(self.open_async()) def set_pump_status(self, status): """ - Updates pump status and logs update to console + Updates pump status and logs update to console. """ self.pump_status = status _logger.info("{} partition {}".format(status, self.lease.partition_id)) def set_lease(self, new_lease): """ - Sets a new partition lease to be processed by the pump + Sets a new partition lease to be processed by the pump. + :param lease: The lease to set. :type lease: ~azure.eventprocessorhost.lease.Lease """ @@ -50,7 +51,7 @@ def set_lease(self, new_lease): async def open_async(self): """ - Opens partition pump + Opens partition pump. """ self.set_pump_status("Opening") self.partition_context = PartitionContext(self.host, self.lease.partition_id, @@ -82,13 +83,15 @@ async def on_open_async(self): def is_closing(self): """ Returns whether pump is closing. - :returns: bool + + :rtype: bool """ return self.pump_status == "Closing" or self.pump_status == "Closed" async def close_async(self, reason): """ Safely closes the pump. + :param reason: The reason for the shutdown. :type reason: str """ @@ -122,6 +125,7 @@ async def close_async(self, reason): async def on_closing_async(self, reason): """ Event handler for on closing event. + :param reason: The reason for the shutdown. :type reason: str """ @@ -130,8 +134,9 @@ async def on_closing_async(self, reason): async def process_events_async(self, events): """ Process pump events. + :param events: List of events to be processed. - :type events: list of ~azure.eventhub.EventData + :type events: list[~azure.eventhub.EventData] """ if events: # Synchronize to serialize calls to the processor. The handler is not installed until @@ -149,6 +154,7 @@ async def process_events_async(self, events): async def process_error_async(self, error): """ Passes error to the event processor for processing. + :param error: An error the occurred. :type error: Exception """ From a82bdf274ed2931d70432ef04da3684b9d13547f Mon Sep 17 00:00:00 2001 From: antisch Date: Mon, 25 Jun 2018 21:52:33 -0700 Subject: [PATCH 05/52] Started iothub support --- .vscode/settings.json | 4 ++ HISTORY.rst | 10 +++++ azure/eventhub/__init__.py | 70 +++++++++++++++++++------------- azure/eventhub/async/__init__.py | 19 ++++----- setup.py | 2 +- 5 files changed, 64 insertions(+), 41 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..1097d23 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,4 @@ +{ + "python.pythonPath": "${workspaceFolder}/env36/bin/python", + "python.linting.enabled": false +} \ No newline at end of file diff --git a/HISTORY.rst b/HISTORY.rst index 4fa8fac..ff1f04f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,16 @@ Release History =============== +0.2.0rc1 (unreleased) ++++++++++++++++++++++ + +- Updated uAMQP dependency to vRC2 +- Added support for constructing IoTHub connections. +- Removed optional `callback` argument from `Receiver.receive` and `AsyncReceiver.receive`. + This removes the potential for messages to be processed via callback for not yet returned + in the batch. + + 0.2.0b2 (2018-05-29) ++++++++++++++++++++ diff --git a/azure/eventhub/__init__.py b/azure/eventhub/__init__.py index 7892358..56ad301 100644 --- a/azure/eventhub/__init__.py +++ b/azure/eventhub/__init__.py @@ -11,11 +11,9 @@ import time import asyncio try: - from urllib import urlparse - from urllib import unquote_plus + from urllib import urlparse, unquote_plus, urlencode except ImportError: - from urllib.parse import unquote_plus - from urllib.parse import urlparse + from urllib.parse import urlparse, unquote_plus, urlencode import uamqp from uamqp import Connection @@ -52,6 +50,29 @@ def _parse_conn_str(conn_str): return endpoint, shared_access_key_name, shared_access_key, entity_path +def _generate_sas_token(uri, policy, key, expiry=None): + """Create a shared access signiture token as a string literal. + :returns: SAS token as string literal. + :rtype: str + """ + from base64 import b64encode, b64decode + from hashlib import sha256 + from hmac import HMAC + if not expiry: + expiry = time.time() + 3600 # Default to 1 hour. + encoded_uri = quote_plus(uri) + ttl = int(expiry) + sign_key = '%s\n%d' % (encoded_uri, ttl) + signature = b64encode(HMAC(b64decode(key), sign_key.encode('utf-8'), sha256).digest()) + result = { + 'sr': uri, + 'sig': signature, + 'se': str(ttl)} + if policy: + result['skn'] = policy + return 'SharedAccessSignature ' + urlencode(result) + + def _build_uri(address, entity): parsed = urlparse(address) if parsed.path: @@ -117,6 +138,14 @@ def from_connection_string(cls, conn_str, eventhub=None, **kwargs): address = _build_uri(address, entity) return cls(address, username=policy, password=key, **kwargs) + @classmethod + def from_iothub_connection_string(cls, conn_str, **kwargs) + address, policy, key, _ = _parse_conn_str(conn_str) + hub_name = address.split('.')[0] + username = "{}@sas.root.{}".format(policy, hub_name) + password = _generate_sas_token(address, policy, key) + return cls(address, username=username, password=password, **kwargs) + def _create_auth(self, auth_uri, username, password): # pylint: disable=no-self-use """ Create an ~uamqp.authentication.SASTokenAuth instance to authenticate @@ -411,7 +440,6 @@ def __init__(self, client, source, prefetch=300, epoch=None): :type epoch: int """ self.offset = None - self._callback = None self.prefetch = prefetch self.epoch = epoch properties = None @@ -437,22 +465,7 @@ def queue_size(self): return self._handler._received_messages.qsize() return 0 - def on_message(self, event): - """ - Callback to process a received message and wrap it in EventData. - Will also call a user supplied callback. - - :param event: The received message. - :type event: ~uamqp.message.Message - :rtype: ~azure.eventhub.EventData. - """ - event_data = EventData(message=event) - if self._callback: - self._callback(event_data) - self.offset = event_data.offset - return event_data - - def receive(self, max_batch_size=None, callback=None, timeout=None): + def receive(self, max_batch_size=None, timeout=None): """ Receive events from the EventHub. @@ -462,20 +475,19 @@ def receive(self, max_batch_size=None, callback=None, timeout=None): retrieve before the time, the result will be empty. If no batch size is supplied, the prefetch size will be the maximum. :type max_batch_size: int - :param callback: A callback to be run for each received event. This must - be a function that accepts a single argument - the event data. This callback - will be run before the message is returned in the result generator. - :type callback: func[~azure.eventhub.EventData] :rtype: list[~azure.eventhub.EventData] """ try: timeout_ms = 1000 * timeout if timeout else 0 - self._callback = callback - batch = self._handler.receive_message_batch( + message_batch = self._handler.receive_message_batch( max_batch_size=max_batch_size, - on_message_received=self.on_message, timeout=timeout_ms) - return batch + data_batch = [] + for message in message_batch: + event_data = EventData(message=event) + self.offset = event_data.offset + data_batch.append(event_data) + return data_batch except errors.AMQPConnectionError as e: message = "Failed to open receiver: {}".format(e) message += "\nPlease check that the partition key is valid " diff --git a/azure/eventhub/async/__init__.py b/azure/eventhub/async/__init__.py index e2e80d5..e03b2e8 100644 --- a/azure/eventhub/async/__init__.py +++ b/azure/eventhub/async/__init__.py @@ -4,7 +4,6 @@ # -------------------------------------------------------------------------------------------- import logging -import queue import asyncio import time import datetime @@ -257,7 +256,6 @@ def __init__(self, client, source, prefetch=300, epoch=None, loop=None): # pyli """ self.loop = loop or asyncio.get_event_loop() self.offset = None - self._callback = None self.prefetch = prefetch self.epoch = epoch properties = None @@ -272,7 +270,7 @@ def __init__(self, client, source, prefetch=300, epoch=None, loop=None): # pyli timeout=self.timeout, loop=self.loop) - async def receive(self, max_batch_size=None, callback=None, timeout=None): + async def receive(self, max_batch_size=None, timeout=None): """ Receive events asynchronously from the EventHub. @@ -282,20 +280,19 @@ async def receive(self, max_batch_size=None, callback=None, timeout=None): retrieve before the time, the result will be empty. If no batch size is supplied, the prefetch size will be the maximum. :type max_batch_size: int - :param callback: A callback to be run for each received event. This must - be a function that accepts a single argument - the event data. This callback - will be run before the message is returned in the result generator. - :type callback: func[~azure.eventhub.EventData] :rtype: list[~azure.eventhub.EventData] """ try: - self._callback = callback timeout_ms = 1000 * timeout if timeout else 0 - batch = await self._handler.receive_message_batch_async( + message_batch = await self._handler.receive_message_batch_async( max_batch_size=max_batch_size, - on_message_received=self.on_message, timeout=timeout_ms) - return batch + data_batch = [] + for message in message_batch: + event_data = EventData(message=event) + self.offset = event_data.offset + data_batch.append(event_data) + return data_batch except errors.AMQPConnectionError as e: message = "Failed to open receiver: {}".format(e) message += "\nPlease check that the partition key is valid " diff --git a/setup.py b/setup.py index 1cc16ce..b6275ea 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ zip_safe=False, packages=find_packages(exclude=["examples", "tests"]), install_requires=[ - 'uamqp==0.1.0rc1', + 'uamqp==0.1.0rc2', 'msrestazure~=0.4.11', 'azure-common~=1.1', 'azure-storage~=0.36.0' From f0e841887dc15f432cb3c6becd617b2b9ec26c8d Mon Sep 17 00:00:00 2001 From: annatisch Date: Tue, 26 Jun 2018 07:49:15 -0700 Subject: [PATCH 06/52] Fixed long running test --- tests/test_longrunning_receive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_longrunning_receive.py b/tests/test_longrunning_receive.py index 6763a24..b5519e3 100644 --- a/tests/test_longrunning_receive.py +++ b/tests/test_longrunning_receive.py @@ -41,7 +41,7 @@ async def pump(_pid, receiver, _args, _dl): print("{}: No events received, queue size {}, delivered {}".format( _pid, receiver.queue_size, - receiver.delivered)) + total)) elif iteration >= 80: iteration = 0 print("{}: total received {}, last sn={}, last offset={}".format( From 808a63880f41bcc8c7dc00c9d6724999d9c85a40 Mon Sep 17 00:00:00 2001 From: annatisch Date: Tue, 26 Jun 2018 09:37:08 -0700 Subject: [PATCH 07/52] Fixed typo and memory leak --- azure/eventhub/__init__.py | 11 +++++------ azure/eventhub/async/__init__.py | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/azure/eventhub/__init__.py b/azure/eventhub/__init__.py index 56ad301..47a37ec 100644 --- a/azure/eventhub/__init__.py +++ b/azure/eventhub/__init__.py @@ -139,7 +139,7 @@ def from_connection_string(cls, conn_str, eventhub=None, **kwargs): return cls(address, username=policy, password=key, **kwargs) @classmethod - def from_iothub_connection_string(cls, conn_str, **kwargs) + def from_iothub_connection_string(cls, conn_str, **kwargs): address, policy, key, _ = _parse_conn_str(conn_str) hub_name = address.split('.')[0] username = "{}@sas.root.{}".format(policy, hub_name) @@ -484,7 +484,7 @@ def receive(self, max_batch_size=None, timeout=None): timeout=timeout_ms) data_batch = [] for message in message_batch: - event_data = EventData(message=event) + event_data = EventData(message=message) self.offset = event_data.offset data_batch.append(event_data) return data_batch @@ -540,15 +540,12 @@ def __init__(self, body=None, batch=None, message=None): self._partition_key = types.AMQPSymbol(EventData.PROP_PARTITION_KEY) self._annotations = {} self._properties = {} - self._header = MessageHeader() - self._header.durable = True if batch: self.message = BatchMessage(data=batch, multi_messages=True) elif message: self.message = message self._annotations = message.annotations self._properties = message.application_properties - self._header = message.header else: if isinstance(body, list) and body: self.message = Message(body[0]) @@ -625,8 +622,10 @@ def partition_key(self, value): """ annotations = dict(self._annotations) annotations[self._partition_key] = value + header = MessageHeader() + header.durable = True self.message.annotations = annotations - self.message.header = self._header + self.message.header = header self._annotations = annotations @property diff --git a/azure/eventhub/async/__init__.py b/azure/eventhub/async/__init__.py index e03b2e8..8e978b5 100644 --- a/azure/eventhub/async/__init__.py +++ b/azure/eventhub/async/__init__.py @@ -289,7 +289,7 @@ async def receive(self, max_batch_size=None, timeout=None): timeout=timeout_ms) data_batch = [] for message in message_batch: - event_data = EventData(message=event) + event_data = EventData(message=message) self.offset = event_data.offset data_batch.append(event_data) return data_batch From 04b7f9e2b9cab111a88d92a34975e1c398ed2950 Mon Sep 17 00:00:00 2001 From: annatisch Date: Mon, 2 Jul 2018 09:50:50 -0700 Subject: [PATCH 08/52] Restructure --- HISTORY.rst | 4 + azure/eventhub/__init__.py | 713 +----------------- azure/eventhub/{async => _async}/__init__.py | 152 +--- azure/eventhub/_async/receiver_async.py | 85 +++ azure/eventhub/_async/sender_async.py | 60 ++ azure/eventhub/client.py | 325 ++++++++ azure/eventhub/common.py | 207 +++++ azure/eventhub/receiver.py | 105 +++ azure/eventhub/sender.py | 99 +++ azure/eventprocessorhost/eh_partition_pump.py | 3 +- azure/eventprocessorhost/partition_manager.py | 2 +- examples/recv_async.py | 3 +- examples/recv_epoch.py | 3 +- examples/send_async.py | 3 +- setup.cfg | 2 - tests/test_negative.py | 8 +- tests/test_receive_async.py | 5 +- tests/test_send_async.py | 4 +- 18 files changed, 930 insertions(+), 853 deletions(-) rename azure/eventhub/{async => _async}/__init__.py (56%) create mode 100644 azure/eventhub/_async/receiver_async.py create mode 100644 azure/eventhub/_async/sender_async.py create mode 100644 azure/eventhub/client.py create mode 100644 azure/eventhub/common.py create mode 100644 azure/eventhub/receiver.py create mode 100644 azure/eventhub/sender.py delete mode 100644 setup.cfg diff --git a/HISTORY.rst b/HISTORY.rst index ff1f04f..3d567d5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,11 +6,15 @@ Release History 0.2.0rc1 (unreleased) +++++++++++++++++++++ +- **Breaking change** Restructured library to support Python 3.7. Submodule `async` has been renamed and all classes from + this module can now be imported from azure.eventhub directly. - Updated uAMQP dependency to vRC2 - Added support for constructing IoTHub connections. - Removed optional `callback` argument from `Receiver.receive` and `AsyncReceiver.receive`. This removes the potential for messages to be processed via callback for not yet returned in the batch. +- Fixed memory leak in receive operations. +- Dropped Python 2.7 wheel support. 0.2.0b2 (2018-05-29) diff --git a/azure/eventhub/__init__.py b/azure/eventhub/__init__.py index 47a37ec..d730dcb 100644 --- a/azure/eventhub/__init__.py +++ b/azure/eventhub/__init__.py @@ -3,708 +3,17 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -import logging -import datetime -import sys -import threading -import uuid -import time -import asyncio -try: - from urllib import urlparse, unquote_plus, urlencode -except ImportError: - from urllib.parse import urlparse, unquote_plus, urlencode - -import uamqp -from uamqp import Connection -from uamqp import SendClient, ReceiveClient -from uamqp import Message, BatchMessage -from uamqp import Source, Target -from uamqp import authentication -from uamqp import constants, types, errors -from uamqp.message import MessageHeader - - __version__ = "0.2.0b2" -log = logging.getLogger(__name__) - - -def _parse_conn_str(conn_str): - endpoint = None - shared_access_key_name = None - shared_access_key = None - entity_path = None - for element in conn_str.split(';'): - key, _, value = element.partition('=') - if key.lower() == 'endpoint': - endpoint = value.rstrip('/') - elif key.lower() == 'sharedaccesskeyname': - shared_access_key_name = value - elif key.lower() == 'sharedaccesskey': - shared_access_key = value - elif key.lower() == 'entitypath': - entity_path = value - if not all([endpoint, shared_access_key_name, shared_access_key]): - raise ValueError("Invalid connection string") - return endpoint, shared_access_key_name, shared_access_key, entity_path - - -def _generate_sas_token(uri, policy, key, expiry=None): - """Create a shared access signiture token as a string literal. - :returns: SAS token as string literal. - :rtype: str - """ - from base64 import b64encode, b64decode - from hashlib import sha256 - from hmac import HMAC - if not expiry: - expiry = time.time() + 3600 # Default to 1 hour. - encoded_uri = quote_plus(uri) - ttl = int(expiry) - sign_key = '%s\n%d' % (encoded_uri, ttl) - signature = b64encode(HMAC(b64decode(key), sign_key.encode('utf-8'), sha256).digest()) - result = { - 'sr': uri, - 'sig': signature, - 'se': str(ttl)} - if policy: - result['skn'] = policy - return 'SharedAccessSignature ' + urlencode(result) - - -def _build_uri(address, entity): - parsed = urlparse(address) - if parsed.path: - return address - if not entity: - raise ValueError("No EventHub specified") - address += "/" + str(entity) - return address - - -class EventHubClient(object): - """ - The EventHubClient class defines a high level interface for sending - events to and receiving events from the Azure Event Hubs service. - """ - - def __init__(self, address, username=None, password=None, debug=False): - """ - Constructs a new EventHubClient with the given address URL. - - :param address: The full URI string of the Event Hub. This can optionally - include URL-encoded access name and key. - :type address: str - :param username: The name of the shared access policy. This must be supplied - if not encoded into the address. - :type username: str - :param password: The shared access key. This must be supplied if not encoded - into the address. - :type password: str - :param debug: Whether to output network trace logs to the logger. Default - is `False`. - :type debug: bool - """ - self.container_id = "eventhub.pysdk-" + str(uuid.uuid4())[:8] - self.address = urlparse(address) - url_username = unquote_plus(self.address.username) if self.address.username else None - username = username or url_username - url_password = unquote_plus(self.address.password) if self.address.password else None - password = password or url_password - if not username or not password: - raise ValueError("Missing username and/or password.") - auth_uri = "sb://{}{}".format(self.address.hostname, self.address.path) - self.auth = self._create_auth(auth_uri, username, password) - self.connection = None - self.debug = debug - - self.clients = [] - self.stopped = False - log.info("{}: Created the Event Hub client".format(self.container_id)) - - @classmethod - def from_connection_string(cls, conn_str, eventhub=None, **kwargs): - """ - Create an EventHubClient from a connection string. - - :param conn_str: The connection string. - :type conn_str: str - :param eventhub: The name of the EventHub, if the EntityName is - not included in the connection string. - """ - address, policy, key, entity = _parse_conn_str(conn_str) - entity = eventhub or entity - address = _build_uri(address, entity) - return cls(address, username=policy, password=key, **kwargs) - - @classmethod - def from_iothub_connection_string(cls, conn_str, **kwargs): - address, policy, key, _ = _parse_conn_str(conn_str) - hub_name = address.split('.')[0] - username = "{}@sas.root.{}".format(policy, hub_name) - password = _generate_sas_token(address, policy, key) - return cls(address, username=username, password=password, **kwargs) - - def _create_auth(self, auth_uri, username, password): # pylint: disable=no-self-use - """ - Create an ~uamqp.authentication.SASTokenAuth instance to authenticate - the session. - - :param auth_uri: The URI to authenticate against. - :type auth_uri: str - :param username: The name of the shared access policy. - :type username: str - :param password: The shared access key. - :type password: str - """ - return authentication.SASTokenAuth.from_shared_access_key(auth_uri, username, password) - - def _create_properties(self): # pylint: disable=no-self-use - """ - Format the properties with which to instantiate the connection. - This acts like a user agent over HTTP. - - :rtype: dict - """ - properties = {} - properties["product"] = "eventhub.python" - properties["version"] = __version__ - properties["framework"] = "Python {}.{}.{}".format(*sys.version_info[0:3]) - properties["platform"] = sys.platform - return properties - - def _create_connection(self): - """ - Create a new ~uamqp.connection.Connection instance that will be shared between all - Sender/Receiver clients. - """ - if not self.connection: - log.info("{}: Creating connection with address={}".format( - self.container_id, self.address.geturl())) - self.connection = Connection( - self.address.hostname, - self.auth, - container_id=self.container_id, - properties=self._create_properties(), - debug=self.debug) - - def _close_connection(self): - """ - Close and destroy the connection. - """ - if self.connection: - self.connection.destroy() - self.connection = None - - def _close_clients(self): - """ - Close all open Sender/Receiver clients. - """ - for client in self.clients: - client.close() - - def run(self): - """ - Run the EventHubClient in blocking mode. - Opens the connection and starts running all Sender/Receiver clients. - - :rtype: ~azure.eventhub.EventHubClient - """ - log.info("{}: Starting {} clients".format(self.container_id, len(self.clients))) - self._create_connection() - for client in self.clients: - client.open(connection=self.connection) - return self - - def stop(self): - """ - Stop the EventHubClient and all its Sender/Receiver clients. - """ - log.info("{}: Stopping {} clients".format(self.container_id, len(self.clients))) - self.stopped = True - self._close_clients() - self._close_connection() - - def get_eventhub_info(self): - """ - Get details on the specified EventHub. - Keys in the details dictionary include: - -'name' - -'type' - -'created_at' - -'partition_count' - -'partition_ids' - - :rtype: dict - """ - self._create_connection() - eh_name = self.address.path.lstrip('/') - target = "amqps://{}/{}".format(self.address.hostname, eh_name) - mgmt_client = uamqp.AMQPClient(target, auth=self.auth, debug=self.debug) - mgmt_client.open(self.connection) - try: - mgmt_msg = Message(application_properties={'name': eh_name}) - response = mgmt_client.mgmt_request( - mgmt_msg, - constants.READ_OPERATION, - op_type=b'com.microsoft:eventhub', - status_code_field=b'status-code', - description_fields=b'status-description') - eh_info = response.get_data() - output = {} - if eh_info: - output['name'] = eh_info[b'name'].decode('utf-8') - output['type'] = eh_info[b'type'].decode('utf-8') - output['created_at'] = datetime.datetime.fromtimestamp(float(eh_info[b'created_at'])/1000) - output['partition_count'] = eh_info[b'partition_count'] - output['partition_ids'] = [p.decode('utf-8') for p in eh_info[b'partition_ids']] - return output - except: - raise - finally: - mgmt_client.close() - - def add_receiver(self, consumer_group, partition, offset=None, prefetch=300): - """ - Add a receiver to the client for a particular consumer group and partition. - - :param consumer_group: The name of the consumer group. - :type consumer_group: str - :param partition: The ID of the partition. - :type partition: str - :param offset: The offset from which to start receiving. - :type offset: ~azure.eventhub.Offset - :param prefetch: The message prefetch count of the receiver. Default is 300. - :type prefetch: int - :rtype: ~azure.eventhub.Receiver - """ - source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( - self.address.hostname, self.address.path, consumer_group, partition) - source = Source(source_url) - if offset is not None: - source.set_filter(offset.selector()) - handler = Receiver(self, source, prefetch=prefetch) - self.clients.append(handler._handler) # pylint: disable=protected-access - return handler - - def add_epoch_receiver(self, consumer_group, partition, epoch, prefetch=300): - """ - Add a receiver to the client with an epoch value. Only a single epoch receiver - can connect to a partition at any given time - additional epoch receivers must have - a higher epoch value or they will be rejected. If a 2nd epoch receiver has - connected, the first will be closed. - - :param consumer_group: The name of the consumer group. - :type consumer_group: str - :param partition: The ID of the partition. - :type partition: str - :param epoch: The epoch value for the receiver. - :type epoch: int - :param prefetch: The message prefetch count of the receiver. Default is 300. - :type prefetch: int - :rtype: ~azure.eventhub.Receiver - """ - source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( - self.address.hostname, self.address.path, consumer_group, partition) - handler = Receiver(self, source_url, prefetch=prefetch, epoch=epoch) - self.clients.append(handler._handler) # pylint: disable=protected-access - return handler - - def add_sender(self, partition=None): - """ - Add a sender to the client to send ~azure.eventhub.EventData object - to an EventHub. - - :param partition: Optionally specify a particular partition to send to. - If omitted, the events will be distributed to available partitions via - round-robin. - :type parition: str - :rtype: ~azure.eventhub.Sender - """ - target = "amqps://{}{}".format(self.address.hostname, self.address.path) - handler = Sender(self, target, partition=partition) - self.clients.append(handler._handler) # pylint: disable=protected-access - return handler - - -class Sender: - """ - Implements a Sender. - """ - TIMEOUT = 60.0 - - def __init__(self, client, target, partition=None): - """ - Instantiate an EventHub event Sender client. - - :param client: The parent EventHubClient. - :type client: ~azure.eventhub.EventHubClient. - :param target: The URI of the EventHub to send to. - :type target: str - """ - self.partition = partition - if partition: - target += "/Partitions/" + partition - self._handler = SendClient( - target, - auth=client.auth, - debug=client.debug, - msg_timeout=Sender.TIMEOUT) - self._outcome = None - self._condition = None - - def send(self, event_data): - """ - Sends an event data and blocks until acknowledgement is - received or operation times out. - - :param event_data: The event to be sent. - :type event_data: ~azure.eventhub.EventData - :raises: ~azure.eventhub.EventHubError if the message fails to - send. - :return: The outcome of the message send. - :rtype: ~uamqp.constants.MessageSendResult - """ - if event_data.partition_key and self.partition: - raise ValueError("EventData partition key cannot be used with a partition sender.") - event_data.message.on_send_complete = self._on_outcome - try: - self._handler.send_message(event_data.message) - if self._outcome != constants.MessageSendResult.Ok: - raise Sender._error(self._outcome, self._condition) - except Exception as e: - raise EventHubError("Send failed: {}".format(e)) - else: - return self._outcome - - def transfer(self, event_data, callback=None): - """ - Transfers an event data and notifies the callback when the operation is done. - - :param event_data: The event to be sent. - :type event_data: ~azure.eventhub.EventData - :param callback: Callback to be run once the message has been send. - This must be a function that accepts two arguments. - :type callback: func[~uamqp.constants.MessageSendResult, ~azure.eventhub.EventHubError] - """ - if event_data.partition_key and self.partition: - raise ValueError("EventData partition key cannot be used with a partition sender.") - if callback: - event_data.message.on_send_complete = lambda o, c: callback(o, Sender._error(o, c)) - self._handler.queue_message(event_data.message) - - def wait(self): - """ - Wait until all transferred events have been sent. - """ - try: - self._handler.wait() - except Exception as e: - raise EventHubError("Send failed: {}".format(e)) +from azure.eventhub.common import EventData, EventHubError, Offset +from azure.eventhub.client import EventHubClient +from azure.eventhub.sender import Sender +from azure.eventhub.receiver import Receiver - def _on_outcome(self, outcome, condition): - """ - Called when the outcome is received for a delivery. - - :param outcome: The outcome of the message delivery - success or failure. - :type outcome: ~uamqp.constants.MessageSendResult - """ - self._outcome = outcome - self._condition = condition - - @staticmethod - def _error(outcome, condition): - return None if outcome == constants.MessageSendResult.Ok else EventHubError(outcome, condition) - - -class Receiver: - """ - Implements a Receiver. - """ - timeout = 0 - _epoch = b'com.microsoft:epoch' - - def __init__(self, client, source, prefetch=300, epoch=None): - """ - Instantiate a receiver. - - :param client: The parent EventHubClient. - :type client: ~azure.eventhub.EventHubClient - :param source: The source EventHub from which to receive events. - :type source: ~uamqp.address.Source - :param prefetch: The number of events to prefetch from the service - for processing. Default is 300. - :type prefetch: int - :param epoch: An optional epoch value. - :type epoch: int - """ - self.offset = None - self.prefetch = prefetch - self.epoch = epoch - properties = None - if epoch: - properties = {types.AMQPSymbol(self._epoch): types.AMQPLong(int(epoch))} - self._handler = ReceiveClient( - source, - auth=client.auth, - debug=client.debug, - prefetch=self.prefetch, - link_properties=properties, - timeout=self.timeout) - - @property - def queue_size(self): - """ - The current size of the unprocessed message queue. - - :rtype: int - """ - # pylint: disable=protected-access - if self._handler._received_messages: - return self._handler._received_messages.qsize() - return 0 - - def receive(self, max_batch_size=None, timeout=None): - """ - Receive events from the EventHub. - - :param max_batch_size: Receive a batch of events. Batch size will - be up to the maximum specified, but will return as soon as service - returns no new events. If combined with a timeout and no events are - retrieve before the time, the result will be empty. If no batch - size is supplied, the prefetch size will be the maximum. - :type max_batch_size: int - :rtype: list[~azure.eventhub.EventData] - """ - try: - timeout_ms = 1000 * timeout if timeout else 0 - message_batch = self._handler.receive_message_batch( - max_batch_size=max_batch_size, - timeout=timeout_ms) - data_batch = [] - for message in message_batch: - event_data = EventData(message=message) - self.offset = event_data.offset - data_batch.append(event_data) - return data_batch - except errors.AMQPConnectionError as e: - message = "Failed to open receiver: {}".format(e) - message += "\nPlease check that the partition key is valid " - if self.epoch: - message += ("and that a higher epoch receiver is not " - "already running for this partition.") - else: - message += ("and whether an epoch receiver is " - "already running for this partition.") - raise EventHubError(message) - except Exception as e: - raise EventHubError("Receive failed: {}".format(e)) - - def selector(self, default): - """ - Create a selector for the current offset if it is set. - - :param default: The fallback receive offset. - :type default: ~azure.eventhub.Offset - :rtype: ~azure.eventhub.Offset - """ - if self.offset is not None: - return Offset(self.offset).selector() - return default - - -class EventData(object): - """ - The EventData class is a holder of event content. - Acts as a wrapper to an ~uamqp.message.Message object. - """ - - PROP_SEQ_NUMBER = b"x-opt-sequence-number" - PROP_OFFSET = b"x-opt-offset" - PROP_PARTITION_KEY = b"x-opt-partition-key" - PROP_TIMESTAMP = b"x-opt-enqueued-time" - PROP_DEVICE_ID = b"iothub-connection-device-id" - - def __init__(self, body=None, batch=None, message=None): - """ - Initialize EventData. - - :param body: The data to send in a single message. - :type body: str, bytes or list - :param batch: A data generator to send batched messages. - :type batch: Generator - :param message: The received message. - :type message: ~uamqp.message.Message - """ - self._partition_key = types.AMQPSymbol(EventData.PROP_PARTITION_KEY) - self._annotations = {} - self._properties = {} - if batch: - self.message = BatchMessage(data=batch, multi_messages=True) - elif message: - self.message = message - self._annotations = message.annotations - self._properties = message.application_properties - else: - if isinstance(body, list) and body: - self.message = Message(body[0]) - for more in body[1:]: - self.message._body.append(more) # pylint: disable=protected-access - elif body is None: - raise ValueError("EventData cannot be None.") - else: - self.message = Message(body) - - - @property - def sequence_number(self): - """ - The sequence number of the event data object. - - :rtype: int - """ - return self._annotations.get(EventData.PROP_SEQ_NUMBER, None) - - @property - def offset(self): - """ - The offset of the event data object. - - :rtype: int - """ - try: - return self._annotations[EventData.PROP_OFFSET].decode('UTF-8') - except (KeyError, AttributeError): - return None - - @property - def enqueued_time(self): - """ - The enqueued timestamp of the event data object. - - :rtype: datetime.datetime - """ - timestamp = self._annotations.get(EventData.PROP_TIMESTAMP, None) - if timestamp: - return datetime.datetime.fromtimestamp(float(timestamp)/1000) - return None - - @property - def device_id(self): - """ - The device ID of the event data object. This is only used for - IoT Hub implementations. - - :rtype: bytes - """ - return self._annotations.get(EventData.PROP_DEVICE_ID, None) - - @property - def partition_key(self): - """ - The partition key of the event data object. - - :rtype: bytes - """ - try: - return self._annotations[self._partition_key] - except KeyError: - return self._annotations.get(EventData.PROP_PARTITION_KEY, None) - - @partition_key.setter - def partition_key(self, value): - """ - Set the partition key of the event data object. - - :param value: The partition key to set. - :type value: str or bytes - """ - annotations = dict(self._annotations) - annotations[self._partition_key] = value - header = MessageHeader() - header.durable = True - self.message.annotations = annotations - self.message.header = header - self._annotations = annotations - - @property - def properties(self): - """ - Application defined properties on the message. - - :rtype: dict - """ - return self._properties - - @properties.setter - def properties(self, value): - """ - Application defined properties on the message. - - :param value: The application properties for the EventData. - :type value: dict - """ - self._properties = value - properties = dict(self._properties) - self.message.application_properties = properties - - @property - def body(self): - """ - The body of the event data object. - - :rtype: bytes or generator[bytes] - """ - return self.message.get_data() - - -class Offset(object): - """ - The offset (position or timestamp) where a receiver starts. Examples: - Beginning of the event stream: - >>> offset = Offset("-1") - End of the event stream: - >>> offset = Offset("@latest") - Events after the specified offset: - >>> offset = Offset("12345") - Events from the specified offset: - >>> offset = Offset("12345", True) - Events after a datetime: - >>> offset = Offset(datetime.datetime.utcnow()) - Events after a specific sequence number: - >>> offset = Offset(1506968696002) - """ - - def __init__(self, value, inclusive=False): - """ - Initialize Offset. - - :param value: The offset value. - :type value: ~datetime.datetime or int or str - :param inclusive: Whether to include the supplied value as the start point. - :type inclusive: bool - """ - self.value = value - self.inclusive = inclusive - - def selector(self): - """ - Creates a selector expression of the offset. - - :rtype: bytes - """ - operator = ">=" if self.inclusive else ">" - if isinstance(self.value, datetime.datetime): - timestamp = (time.mktime(self.value.timetuple()) * 1000) + (self.value.microsecond/1000) - return ("amqp.annotation.x-opt-enqueued-time {} '{}'".format(operator, int(timestamp))).encode('utf-8') - elif isinstance(self.value, int): - return ("amqp.annotation.x-opt-sequence-number {} '{}'".format(operator, self.value)).encode('utf-8') - return ("amqp.annotation.x-opt-offset {} '{}'".format(operator, self.value)).encode('utf-8') - - -class EventHubError(Exception): - """ - Represents an error happened in the client. - """ - pass +try: + from azure.eventhub._async import ( + EventHubClientAsync, + AsyncSender, + AsyncReceiver) +except (ImportError, SyntaxError): + pass # Python 3 async features not supported diff --git a/azure/eventhub/async/__init__.py b/azure/eventhub/_async/__init__.py similarity index 56% rename from azure/eventhub/async/__init__.py rename to azure/eventhub/_async/__init__.py index 8e978b5..5831051 100644 --- a/azure/eventhub/async/__init__.py +++ b/azure/eventhub/_async/__init__.py @@ -8,12 +8,16 @@ import time import datetime -from uamqp.async import SASTokenAsync -from uamqp.async import ConnectionAsync -from uamqp import Message, AMQPClientAsync, SendClientAsync, ReceiveClientAsync, Source from uamqp import constants, types, errors +from uamqp.authentication import SASTokenAsync +from uamqp import ( + Message, + Source, + ConnectionAsync, + AMQPClientAsync, + SendClientAsync, + ReceiveClientAsync) -from azure import eventhub from azure.eventhub import ( Sender, Receiver, @@ -21,6 +25,9 @@ EventData, EventHubError) +from .sender_async import AsyncSender +from .receiver_async import AsyncReceiver + log = logging.getLogger(__name__) @@ -33,7 +40,7 @@ class EventHubClientAsync(EventHubClient): def _create_auth(self, auth_uri, username, password): # pylint: disable=no-self-use """ - Create an ~uamqp.async.authentication_async.SASTokenAuthAsync instance to authenticate + Create an ~uamqp.authentication.cbs_auth_async.SASTokenAuthAsync instance to authenticate the session. :param auth_uri: The URI to authenticate against. @@ -47,7 +54,7 @@ def _create_auth(self, auth_uri, username, password): # pylint: disable=no-self def _create_connection_async(self): """ - Create a new ~uamqp.async.connection_async.ConnectionAsync instance that will be shared between all + Create a new ~uamqp._async.connection_async.ConnectionAsync instance that will be shared between all AsyncSender/AsyncReceiver clients. """ if not self.connection: @@ -80,7 +87,7 @@ async def run_async(self): Run the EventHubClient asynchronously. Opens the connection and starts running all AsyncSender/AsyncReceiver clients. - :rtype: ~azure.eventhub.async.EventHubClientAsync + :rtype: ~azure.eventhub._async.EventHubClientAsync """ log.info("{}: Starting {} clients".format(self.container_id, len(self.clients))) self._create_connection_async() @@ -132,10 +139,10 @@ def add_async_receiver(self, consumer_group, partition, offset=None, prefetch=30 :param partition: The ID of the partition. :type partition: str :param offset: The offset from which to start receiving. - :type offset: ~azure.eventhub.Offset + :type offset: ~azure.eventhub.common.Offset :param prefetch: The message prefetch count of the receiver. Default is 300. :type prefetch: int - :rtype: ~azure.eventhub.async.ReceiverAsync + :rtype: ~azure.eventhub._async.receiver_async.ReceiverAsync """ source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( self.address.hostname, self.address.path, consumer_group, partition) @@ -161,7 +168,7 @@ def add_async_epoch_receiver(self, consumer_group, partition, epoch, prefetch=30 :type epoch: int :param prefetch: The message prefetch count of the receiver. Default is 300. :type prefetch: int - :rtype: ~azure.eventhub.async.ReceiverAsync + :rtype: ~azure.eventhub._async.receiver_async.ReceiverAsync """ source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( self.address.hostname, self.address.path, consumer_group, partition) @@ -171,137 +178,16 @@ def add_async_epoch_receiver(self, consumer_group, partition, epoch, prefetch=30 def add_async_sender(self, partition=None, loop=None): """ - Add an async sender to the client to send ~azure.eventhub.EventData object + Add an async sender to the client to send ~azure.eventhub.common.EventData object to an EventHub. :param partition: Optionally specify a particular partition to send to. If omitted, the events will be distributed to available partitions via round-robin. :type partition: str - :rtype: ~azure.eventhub.async.SenderAsync + :rtype: ~azure.eventhub._async.sender_async.SenderAsync """ target = "amqps://{}{}".format(self.address.hostname, self.address.path) handler = AsyncSender(self, target, partition=partition, loop=loop) self.clients.append(handler._handler) # pylint: disable=protected-access return handler - -class AsyncSender(Sender): - """ - Implements the async API of a Sender. - """ - - def __init__(self, client, target, partition=None, loop=None): # pylint: disable=super-init-not-called - """ - Instantiate an EventHub event SenderAsync client. - - :param client: The parent EventHubClientAsync. - :type client: ~azure.eventhub.async.EventHubClientAsync - :param target: The URI of the EventHub to send to. - :type target: str - :param loop: An event loop. - """ - self.partition = partition - if partition: - target += "/Partitions/" + partition - self.loop = loop or asyncio.get_event_loop() - self._handler = SendClientAsync( - target, - auth=client.auth, - debug=client.debug, - msg_timeout=Sender.TIMEOUT, - loop=self.loop) - self._outcome = None - self._condition = None - - async def send(self, event_data): - """ - Sends an event data and asynchronously waits until - acknowledgement is received or operation times out. - - :param event_data: The event to be sent. - :type event_data: ~azure.eventhub.EventData - :raises: ~azure.eventhub.EventHubError if the message fails to - send. - """ - if event_data.partition_key and self.partition: - raise ValueError("EventData partition key cannot be used with a partition sender.") - event_data.message.on_send_complete = self._on_outcome - try: - await self._handler.send_message_async(event_data.message) - if self._outcome != constants.MessageSendResult.Ok: - raise Sender._error(self._outcome, self._condition) - except Exception as e: - raise EventHubError("Send failed: {}".format(e)) - - -class AsyncReceiver(Receiver): - """ - Implements the async API of a Receiver. - """ - - def __init__(self, client, source, prefetch=300, epoch=None, loop=None): # pylint: disable=super-init-not-called - """ - Instantiate an async receiver. - - :param client: The parent EventHubClientAsync. - :type client: ~azure.eventhub.async.EventHubClientAsync - :param source: The source EventHub from which to receive events. - :type source: ~uamqp.address.Source - :param prefetch: The number of events to prefetch from the service - for processing. Default is 300. - :type prefetch: int - :param epoch: An optional epoch value. - :type epoch: int - :param loop: An event loop. - """ - self.loop = loop or asyncio.get_event_loop() - self.offset = None - self.prefetch = prefetch - self.epoch = epoch - properties = None - if epoch: - properties = {types.AMQPSymbol(self._epoch): types.AMQPLong(int(epoch))} - self._handler = ReceiveClientAsync( - source, - auth=client.auth, - debug=client.debug, - prefetch=self.prefetch, - link_properties=properties, - timeout=self.timeout, - loop=self.loop) - - async def receive(self, max_batch_size=None, timeout=None): - """ - Receive events asynchronously from the EventHub. - - :param max_batch_size: Receive a batch of events. Batch size will - be up to the maximum specified, but will return as soon as service - returns no new events. If combined with a timeout and no events are - retrieve before the time, the result will be empty. If no batch - size is supplied, the prefetch size will be the maximum. - :type max_batch_size: int - :rtype: list[~azure.eventhub.EventData] - """ - try: - timeout_ms = 1000 * timeout if timeout else 0 - message_batch = await self._handler.receive_message_batch_async( - max_batch_size=max_batch_size, - timeout=timeout_ms) - data_batch = [] - for message in message_batch: - event_data = EventData(message=message) - self.offset = event_data.offset - data_batch.append(event_data) - return data_batch - except errors.AMQPConnectionError as e: - message = "Failed to open receiver: {}".format(e) - message += "\nPlease check that the partition key is valid " - if self.epoch: - message += "and that a higher epoch receiver is " \ - "not already running for this partition." - else: - message += "and whether an epoch receiver is " \ - "already running for this partition." - raise EventHubError(message) - except Exception as e: - raise EventHubError("Receive failed: {}".format(e)) diff --git a/azure/eventhub/_async/receiver_async.py b/azure/eventhub/_async/receiver_async.py new file mode 100644 index 0000000..3c044d4 --- /dev/null +++ b/azure/eventhub/_async/receiver_async.py @@ -0,0 +1,85 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import asyncio + +from uamqp import errors, types +from uamqp import ReceiveClientAsync + +from azure.eventhub import EventHubError, EventData +from azure.eventhub.receiver import Receiver + + +class AsyncReceiver(Receiver): + """ + Implements the async API of a Receiver. + """ + + def __init__(self, client, source, prefetch=300, epoch=None, loop=None): # pylint: disable=super-init-not-called + """ + Instantiate an async receiver. + + :param client: The parent EventHubClientAsync. + :type client: ~azure.eventhub._async.EventHubClientAsync + :param source: The source EventHub from which to receive events. + :type source: ~uamqp.address.Source + :param prefetch: The number of events to prefetch from the service + for processing. Default is 300. + :type prefetch: int + :param epoch: An optional epoch value. + :type epoch: int + :param loop: An event loop. + """ + self.loop = loop or asyncio.get_event_loop() + self.offset = None + self.prefetch = prefetch + self.epoch = epoch + properties = None + if epoch: + properties = {types.AMQPSymbol(self._epoch): types.AMQPLong(int(epoch))} + self._handler = ReceiveClientAsync( + source, + auth=client.auth, + debug=client.debug, + prefetch=self.prefetch, + link_properties=properties, + timeout=self.timeout, + loop=self.loop) + + async def receive(self, max_batch_size=None, timeout=None): + """ + Receive events asynchronously from the EventHub. + + :param max_batch_size: Receive a batch of events. Batch size will + be up to the maximum specified, but will return as soon as service + returns no new events. If combined with a timeout and no events are + retrieve before the time, the result will be empty. If no batch + size is supplied, the prefetch size will be the maximum. + :type max_batch_size: int + :rtype: list[~azure.eventhub.EventData] + """ + try: + timeout_ms = 1000 * timeout if timeout else 0 + message_batch = await self._handler.receive_message_batch_async( + max_batch_size=max_batch_size, + timeout=timeout_ms) + data_batch = [] + for message in message_batch: + event_data = EventData(message=message) + self.offset = event_data.offset + data_batch.append(event_data) + return data_batch + except errors.AMQPConnectionError as e: + message = "Failed to open receiver: {}".format(e) + message += "\nPlease check that the partition key is valid " + if self.epoch: + message += "and that a higher epoch receiver is " \ + "not already running for this partition." + else: + message += "and whether an epoch receiver is " \ + "already running for this partition." + raise EventHubError(message) + except Exception as e: + raise EventHubError("Receive failed: {}".format(e)) diff --git a/azure/eventhub/_async/sender_async.py b/azure/eventhub/_async/sender_async.py new file mode 100644 index 0000000..b702162 --- /dev/null +++ b/azure/eventhub/_async/sender_async.py @@ -0,0 +1,60 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import asyncio + +from uamqp import constants +from uamqp import SendClientAsync + +from azure.eventhub import EventHubError +from azure.eventhub.sender import Sender + +class AsyncSender(Sender): + """ + Implements the async API of a Sender. + """ + + def __init__(self, client, target, partition=None, loop=None): # pylint: disable=super-init-not-called + """ + Instantiate an EventHub event SenderAsync client. + + :param client: The parent EventHubClientAsync. + :type client: ~azure.eventhub._async.EventHubClientAsync + :param target: The URI of the EventHub to send to. + :type target: str + :param loop: An event loop. + """ + self.partition = partition + if partition: + target += "/Partitions/" + partition + self.loop = loop or asyncio.get_event_loop() + self._handler = SendClientAsync( + target, + auth=client.auth, + debug=client.debug, + msg_timeout=Sender.TIMEOUT, + loop=self.loop) + self._outcome = None + self._condition = None + + async def send(self, event_data): + """ + Sends an event data and asynchronously waits until + acknowledgement is received or operation times out. + + :param event_data: The event to be sent. + :type event_data: ~azure.eventhub.EventData + :raises: ~azure.eventhub.EventHubError if the message fails to + send. + """ + if event_data.partition_key and self.partition: + raise ValueError("EventData partition key cannot be used with a partition sender.") + event_data.message.on_send_complete = self._on_outcome + try: + await self._handler.send_message_async(event_data.message) + if self._outcome != constants.MessageSendResult.Ok: + raise Sender._error(self._outcome, self._condition) + except Exception as e: + raise EventHubError("Send failed: {}".format(e)) diff --git a/azure/eventhub/client.py b/azure/eventhub/client.py new file mode 100644 index 0000000..d0a6687 --- /dev/null +++ b/azure/eventhub/client.py @@ -0,0 +1,325 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import logging +import datetime +import sys +import uuid +import time +try: + from urllib import urlparse, unquote_plus, urlencode, quote_plus +except ImportError: + from urllib.parse import urlparse, unquote_plus, urlencode, quote_plus + +import uamqp +from uamqp import Connection +from uamqp import Message +from uamqp import Source +from uamqp import authentication +from uamqp import constants + +from azure.eventhub import __version__ +from azure.eventhub.sender import Sender +from azure.eventhub.receiver import Receiver + +log = logging.getLogger(__name__) + + +def _parse_conn_str(conn_str): + endpoint = None + shared_access_key_name = None + shared_access_key = None + entity_path = None + for element in conn_str.split(';'): + key, _, value = element.partition('=') + if key.lower() == 'endpoint': + endpoint = value.rstrip('/') + elif key.lower() == 'sharedaccesskeyname': + shared_access_key_name = value + elif key.lower() == 'sharedaccesskey': + shared_access_key = value + elif key.lower() == 'entitypath': + entity_path = value + if not all([endpoint, shared_access_key_name, shared_access_key]): + raise ValueError("Invalid connection string") + return endpoint, shared_access_key_name, shared_access_key, entity_path + + +def _generate_sas_token(uri, policy, key, expiry=None): + """Create a shared access signiture token as a string literal. + :returns: SAS token as string literal. + :rtype: str + """ + from base64 import b64encode, b64decode + from hashlib import sha256 + from hmac import HMAC + if not expiry: + expiry = time.time() + 3600 # Default to 1 hour. + encoded_uri = quote_plus(uri) + ttl = int(expiry) + sign_key = '%s\n%d' % (encoded_uri, ttl) + signature = b64encode(HMAC(b64decode(key), sign_key.encode('utf-8'), sha256).digest()) + result = { + 'sr': uri, + 'sig': signature, + 'se': str(ttl)} + if policy: + result['skn'] = policy + return 'SharedAccessSignature ' + urlencode(result) + + +def _build_uri(address, entity): + parsed = urlparse(address) + if parsed.path: + return address + if not entity: + raise ValueError("No EventHub specified") + address += "/" + str(entity) + return address + + +class EventHubClient(object): + """ + The EventHubClient class defines a high level interface for sending + events to and receiving events from the Azure Event Hubs service. + """ + + def __init__(self, address, username=None, password=None, debug=False): + """ + Constructs a new EventHubClient with the given address URL. + + :param address: The full URI string of the Event Hub. This can optionally + include URL-encoded access name and key. + :type address: str + :param username: The name of the shared access policy. This must be supplied + if not encoded into the address. + :type username: str + :param password: The shared access key. This must be supplied if not encoded + into the address. + :type password: str + :param debug: Whether to output network trace logs to the logger. Default + is `False`. + :type debug: bool + """ + self.container_id = "eventhub.pysdk-" + str(uuid.uuid4())[:8] + self.address = urlparse(address) + url_username = unquote_plus(self.address.username) if self.address.username else None + username = username or url_username + url_password = unquote_plus(self.address.password) if self.address.password else None + password = password or url_password + if not username or not password: + raise ValueError("Missing username and/or password.") + auth_uri = "sb://{}{}".format(self.address.hostname, self.address.path) + self.auth = self._create_auth(auth_uri, username, password) + self.connection = None + self.debug = debug + + self.clients = [] + self.stopped = False + log.info("{}: Created the Event Hub client".format(self.container_id)) + + @classmethod + def from_connection_string(cls, conn_str, eventhub=None, **kwargs): + """ + Create an EventHubClient from a connection string. + + :param conn_str: The connection string. + :type conn_str: str + :param eventhub: The name of the EventHub, if the EntityName is + not included in the connection string. + """ + address, policy, key, entity = _parse_conn_str(conn_str) + entity = eventhub or entity + address = _build_uri(address, entity) + return cls(address, username=policy, password=key, **kwargs) + + @classmethod + def from_iothub_connection_string(cls, conn_str, **kwargs): + address, policy, key, _ = _parse_conn_str(conn_str) + hub_name = address.split('.')[0] + username = "{}@sas.root.{}".format(policy, hub_name) + password = _generate_sas_token(address, policy, key) + return cls(address, username=username, password=password, **kwargs) + + def _create_auth(self, auth_uri, username, password): # pylint: disable=no-self-use + """ + Create an ~uamqp.authentication.SASTokenAuth instance to authenticate + the session. + + :param auth_uri: The URI to authenticate against. + :type auth_uri: str + :param username: The name of the shared access policy. + :type username: str + :param password: The shared access key. + :type password: str + """ + return authentication.SASTokenAuth.from_shared_access_key(auth_uri, username, password) + + def _create_properties(self): # pylint: disable=no-self-use + """ + Format the properties with which to instantiate the connection. + This acts like a user agent over HTTP. + + :rtype: dict + """ + properties = {} + properties["product"] = "eventhub.python" + properties["version"] = __version__ + properties["framework"] = "Python {}.{}.{}".format(*sys.version_info[0:3]) + properties["platform"] = sys.platform + return properties + + def _create_connection(self): + """ + Create a new ~uamqp.connection.Connection instance that will be shared between all + Sender/Receiver clients. + """ + if not self.connection: + log.info("{}: Creating connection with address={}".format( + self.container_id, self.address.geturl())) + self.connection = Connection( + self.address.hostname, + self.auth, + container_id=self.container_id, + properties=self._create_properties(), + debug=self.debug) + + def _close_connection(self): + """ + Close and destroy the connection. + """ + if self.connection: + self.connection.destroy() + self.connection = None + + def _close_clients(self): + """ + Close all open Sender/Receiver clients. + """ + for client in self.clients: + client.close() + + def run(self): + """ + Run the EventHubClient in blocking mode. + Opens the connection and starts running all Sender/Receiver clients. + + :rtype: ~azure.eventhub.EventHubClient + """ + log.info("{}: Starting {} clients".format(self.container_id, len(self.clients))) + self._create_connection() + for client in self.clients: + client.open(connection=self.connection) + return self + + def stop(self): + """ + Stop the EventHubClient and all its Sender/Receiver clients. + """ + log.info("{}: Stopping {} clients".format(self.container_id, len(self.clients))) + self.stopped = True + self._close_clients() + self._close_connection() + + def get_eventhub_info(self): + """ + Get details on the specified EventHub. + Keys in the details dictionary include: + -'name' + -'type' + -'created_at' + -'partition_count' + -'partition_ids' + + :rtype: dict + """ + self._create_connection() + eh_name = self.address.path.lstrip('/') + target = "amqps://{}/{}".format(self.address.hostname, eh_name) + mgmt_client = uamqp.AMQPClient(target, auth=self.auth, debug=self.debug) + mgmt_client.open(self.connection) + try: + mgmt_msg = Message(application_properties={'name': eh_name}) + response = mgmt_client.mgmt_request( + mgmt_msg, + constants.READ_OPERATION, + op_type=b'com.microsoft:eventhub', + status_code_field=b'status-code', + description_fields=b'status-description') + eh_info = response.get_data() + output = {} + if eh_info: + output['name'] = eh_info[b'name'].decode('utf-8') + output['type'] = eh_info[b'type'].decode('utf-8') + output['created_at'] = datetime.datetime.fromtimestamp(float(eh_info[b'created_at'])/1000) + output['partition_count'] = eh_info[b'partition_count'] + output['partition_ids'] = [p.decode('utf-8') for p in eh_info[b'partition_ids']] + return output + except: + raise + finally: + mgmt_client.close() + + def add_receiver(self, consumer_group, partition, offset=None, prefetch=300): + """ + Add a receiver to the client for a particular consumer group and partition. + + :param consumer_group: The name of the consumer group. + :type consumer_group: str + :param partition: The ID of the partition. + :type partition: str + :param offset: The offset from which to start receiving. + :type offset: ~azure.eventhub.Offset + :param prefetch: The message prefetch count of the receiver. Default is 300. + :type prefetch: int + :rtype: ~azure.eventhub.Receiver + """ + source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( + self.address.hostname, self.address.path, consumer_group, partition) + source = Source(source_url) + if offset is not None: + source.set_filter(offset.selector()) + handler = Receiver(self, source, prefetch=prefetch) + self.clients.append(handler._handler) # pylint: disable=protected-access + return handler + + def add_epoch_receiver(self, consumer_group, partition, epoch, prefetch=300): + """ + Add a receiver to the client with an epoch value. Only a single epoch receiver + can connect to a partition at any given time - additional epoch receivers must have + a higher epoch value or they will be rejected. If a 2nd epoch receiver has + connected, the first will be closed. + + :param consumer_group: The name of the consumer group. + :type consumer_group: str + :param partition: The ID of the partition. + :type partition: str + :param epoch: The epoch value for the receiver. + :type epoch: int + :param prefetch: The message prefetch count of the receiver. Default is 300. + :type prefetch: int + :rtype: ~azure.eventhub.Receiver + """ + source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( + self.address.hostname, self.address.path, consumer_group, partition) + handler = Receiver(self, source_url, prefetch=prefetch, epoch=epoch) + self.clients.append(handler._handler) # pylint: disable=protected-access + return handler + + def add_sender(self, partition=None): + """ + Add a sender to the client to send ~azure.eventhub.EventData object + to an EventHub. + + :param partition: Optionally specify a particular partition to send to. + If omitted, the events will be distributed to available partitions via + round-robin. + :type parition: str + :rtype: ~azure.eventhub.Sender + """ + target = "amqps://{}{}".format(self.address.hostname, self.address.path) + handler = Sender(self, target, partition=partition) + self.clients.append(handler._handler) # pylint: disable=protected-access + return handler diff --git a/azure/eventhub/common.py b/azure/eventhub/common.py new file mode 100644 index 0000000..f0a7f92 --- /dev/null +++ b/azure/eventhub/common.py @@ -0,0 +1,207 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import datetime +import time + +from uamqp import Message, BatchMessage +from uamqp import types +from uamqp.message import MessageHeader + + +class EventData(object): + """ + The EventData class is a holder of event content. + Acts as a wrapper to an ~uamqp.message.Message object. + """ + + PROP_SEQ_NUMBER = b"x-opt-sequence-number" + PROP_OFFSET = b"x-opt-offset" + PROP_PARTITION_KEY = b"x-opt-partition-key" + PROP_TIMESTAMP = b"x-opt-enqueued-time" + PROP_DEVICE_ID = b"iothub-connection-device-id" + + def __init__(self, body=None, batch=None, message=None): + """ + Initialize EventData. + + :param body: The data to send in a single message. + :type body: str, bytes or list + :param batch: A data generator to send batched messages. + :type batch: Generator + :param message: The received message. + :type message: ~uamqp.message.Message + """ + self._partition_key = types.AMQPSymbol(EventData.PROP_PARTITION_KEY) + self._annotations = {} + self._properties = {} + if batch: + self.message = BatchMessage(data=batch, multi_messages=True) + elif message: + self.message = message + self._annotations = message.annotations + self._properties = message.application_properties + else: + if isinstance(body, list) and body: + self.message = Message(body[0]) + for more in body[1:]: + self.message._body.append(more) # pylint: disable=protected-access + elif body is None: + raise ValueError("EventData cannot be None.") + else: + self.message = Message(body) + + + @property + def sequence_number(self): + """ + The sequence number of the event data object. + + :rtype: int + """ + return self._annotations.get(EventData.PROP_SEQ_NUMBER, None) + + @property + def offset(self): + """ + The offset of the event data object. + + :rtype: int + """ + try: + return self._annotations[EventData.PROP_OFFSET].decode('UTF-8') + except (KeyError, AttributeError): + return None + + @property + def enqueued_time(self): + """ + The enqueued timestamp of the event data object. + + :rtype: datetime.datetime + """ + timestamp = self._annotations.get(EventData.PROP_TIMESTAMP, None) + if timestamp: + return datetime.datetime.fromtimestamp(float(timestamp)/1000) + return None + + @property + def device_id(self): + """ + The device ID of the event data object. This is only used for + IoT Hub implementations. + + :rtype: bytes + """ + return self._annotations.get(EventData.PROP_DEVICE_ID, None) + + @property + def partition_key(self): + """ + The partition key of the event data object. + + :rtype: bytes + """ + try: + return self._annotations[self._partition_key] + except KeyError: + return self._annotations.get(EventData.PROP_PARTITION_KEY, None) + + @partition_key.setter + def partition_key(self, value): + """ + Set the partition key of the event data object. + + :param value: The partition key to set. + :type value: str or bytes + """ + annotations = dict(self._annotations) + annotations[self._partition_key] = value + header = MessageHeader() + header.durable = True + self.message.annotations = annotations + self.message.header = header + self._annotations = annotations + + @property + def properties(self): + """ + Application defined properties on the message. + + :rtype: dict + """ + return self._properties + + @properties.setter + def properties(self, value): + """ + Application defined properties on the message. + + :param value: The application properties for the EventData. + :type value: dict + """ + self._properties = value + properties = dict(self._properties) + self.message.application_properties = properties + + @property + def body(self): + """ + The body of the event data object. + + :rtype: bytes or generator[bytes] + """ + return self.message.get_data() + + +class Offset(object): + """ + The offset (position or timestamp) where a receiver starts. Examples: + Beginning of the event stream: + >>> offset = Offset("-1") + End of the event stream: + >>> offset = Offset("@latest") + Events after the specified offset: + >>> offset = Offset("12345") + Events from the specified offset: + >>> offset = Offset("12345", True) + Events after a datetime: + >>> offset = Offset(datetime.datetime.utcnow()) + Events after a specific sequence number: + >>> offset = Offset(1506968696002) + """ + + def __init__(self, value, inclusive=False): + """ + Initialize Offset. + + :param value: The offset value. + :type value: ~datetime.datetime or int or str + :param inclusive: Whether to include the supplied value as the start point. + :type inclusive: bool + """ + self.value = value + self.inclusive = inclusive + + def selector(self): + """ + Creates a selector expression of the offset. + + :rtype: bytes + """ + operator = ">=" if self.inclusive else ">" + if isinstance(self.value, datetime.datetime): + timestamp = (time.mktime(self.value.timetuple()) * 1000) + (self.value.microsecond/1000) + return ("amqp.annotation.x-opt-enqueued-time {} '{}'".format(operator, int(timestamp))).encode('utf-8') + elif isinstance(self.value, int): + return ("amqp.annotation.x-opt-sequence-number {} '{}'".format(operator, self.value)).encode('utf-8') + return ("amqp.annotation.x-opt-offset {} '{}'".format(operator, self.value)).encode('utf-8') + + +class EventHubError(Exception): + """ + Represents an error happened in the client. + """ + pass diff --git a/azure/eventhub/receiver.py b/azure/eventhub/receiver.py new file mode 100644 index 0000000..c332bca --- /dev/null +++ b/azure/eventhub/receiver.py @@ -0,0 +1,105 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from uamqp import types, errors +from uamqp import ReceiveClient + +from azure.eventhub.common import EventHubError, EventData, Offset + + +class Receiver: + """ + Implements a Receiver. + """ + timeout = 0 + _epoch = b'com.microsoft:epoch' + + def __init__(self, client, source, prefetch=300, epoch=None): + """ + Instantiate a receiver. + + :param client: The parent EventHubClient. + :type client: ~azure.eventhub.EventHubClient + :param source: The source EventHub from which to receive events. + :type source: ~uamqp.address.Source + :param prefetch: The number of events to prefetch from the service + for processing. Default is 300. + :type prefetch: int + :param epoch: An optional epoch value. + :type epoch: int + """ + self.offset = None + self.prefetch = prefetch + self.epoch = epoch + properties = None + if epoch: + properties = {types.AMQPSymbol(self._epoch): types.AMQPLong(int(epoch))} + self._handler = ReceiveClient( + source, + auth=client.auth, + debug=client.debug, + prefetch=self.prefetch, + link_properties=properties, + timeout=self.timeout) + + @property + def queue_size(self): + """ + The current size of the unprocessed message queue. + + :rtype: int + """ + # pylint: disable=protected-access + if self._handler._received_messages: + return self._handler._received_messages.qsize() + return 0 + + def receive(self, max_batch_size=None, timeout=None): + """ + Receive events from the EventHub. + + :param max_batch_size: Receive a batch of events. Batch size will + be up to the maximum specified, but will return as soon as service + returns no new events. If combined with a timeout and no events are + retrieve before the time, the result will be empty. If no batch + size is supplied, the prefetch size will be the maximum. + :type max_batch_size: int + :rtype: list[~azure.eventhub.EventData] + """ + try: + timeout_ms = 1000 * timeout if timeout else 0 + message_batch = self._handler.receive_message_batch( + max_batch_size=max_batch_size, + timeout=timeout_ms) + data_batch = [] + for message in message_batch: + event_data = EventData(message=message) + self.offset = event_data.offset + data_batch.append(event_data) + return data_batch + except errors.AMQPConnectionError as e: + message = "Failed to open receiver: {}".format(e) + message += "\nPlease check that the partition key is valid " + if self.epoch: + message += ("and that a higher epoch receiver is not " + "already running for this partition.") + else: + message += ("and whether an epoch receiver is " + "already running for this partition.") + raise EventHubError(message) + except Exception as e: + raise EventHubError("Receive failed: {}".format(e)) + + def selector(self, default): + """ + Create a selector for the current offset if it is set. + + :param default: The fallback receive offset. + :type default: ~azure.eventhub.Offset + :rtype: ~azure.eventhub.Offset + """ + if self.offset is not None: + return Offset(self.offset).selector() + return default diff --git a/azure/eventhub/sender.py b/azure/eventhub/sender.py new file mode 100644 index 0000000..78c2195 --- /dev/null +++ b/azure/eventhub/sender.py @@ -0,0 +1,99 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from uamqp import constants +from uamqp import SendClient + +from azure.eventhub.common import EventHubError + + +class Sender: + """ + Implements a Sender. + """ + TIMEOUT = 60.0 + + def __init__(self, client, target, partition=None): + """ + Instantiate an EventHub event Sender client. + + :param client: The parent EventHubClient. + :type client: ~azure.eventhub.EventHubClient. + :param target: The URI of the EventHub to send to. + :type target: str + """ + self.partition = partition + if partition: + target += "/Partitions/" + partition + self._handler = SendClient( + target, + auth=client.auth, + debug=client.debug, + msg_timeout=Sender.TIMEOUT) + self._outcome = None + self._condition = None + + def send(self, event_data): + """ + Sends an event data and blocks until acknowledgement is + received or operation times out. + + :param event_data: The event to be sent. + :type event_data: ~azure.eventhub.client.EventData + :raises: ~azure.eventhub.client.EventHubError if the message fails to + send. + :return: The outcome of the message send. + :rtype: ~uamqp.constants.MessageSendResult + """ + if event_data.partition_key and self.partition: + raise ValueError("EventData partition key cannot be used with a partition sender.") + event_data.message.on_send_complete = self._on_outcome + try: + self._handler.send_message(event_data.message) + if self._outcome != constants.MessageSendResult.Ok: + raise Sender._error(self._outcome, self._condition) + except Exception as e: + raise EventHubError("Send failed: {}".format(e)) + else: + return self._outcome + + def transfer(self, event_data, callback=None): + """ + Transfers an event data and notifies the callback when the operation is done. + + :param event_data: The event to be sent. + :type event_data: ~azure.eventhub.client.EventData + :param callback: Callback to be run once the message has been send. + This must be a function that accepts two arguments. + :type callback: func[~uamqp.constants.MessageSendResult, ~azure.eventhub.client.EventHubError] + """ + if event_data.partition_key and self.partition: + raise ValueError("EventData partition key cannot be used with a partition sender.") + if callback: + event_data.message.on_send_complete = lambda o, c: callback(o, Sender._error(o, c)) + self._handler.queue_message(event_data.message) + + def wait(self): + """ + Wait until all transferred events have been sent. + """ + try: + self._handler.wait() + except Exception as e: + raise EventHubError("Send failed: {}".format(e)) + + def _on_outcome(self, outcome, condition): + """ + Called when the outcome is received for a delivery. + + :param outcome: The outcome of the message delivery - success or failure. + :type outcome: ~uamqp.constants.MessageSendResult + """ + self._outcome = outcome + self._condition = condition + + @staticmethod + def _error(outcome, condition): + return None if outcome == constants.MessageSendResult.Ok else EventHubError(outcome, condition) diff --git a/azure/eventprocessorhost/eh_partition_pump.py b/azure/eventprocessorhost/eh_partition_pump.py index 738422d..1c1b813 100644 --- a/azure/eventprocessorhost/eh_partition_pump.py +++ b/azure/eventprocessorhost/eh_partition_pump.py @@ -5,8 +5,7 @@ import logging import asyncio -from azure.eventhub import Offset -from azure.eventhub.async import EventHubClientAsync +from azure.eventhub import Offset, EventHubClientAsync from azure.eventprocessorhost.partition_pump import PartitionPump diff --git a/azure/eventprocessorhost/partition_manager.py b/azure/eventprocessorhost/partition_manager.py index 2580abc..2ae402e 100644 --- a/azure/eventprocessorhost/partition_manager.py +++ b/azure/eventprocessorhost/partition_manager.py @@ -8,7 +8,7 @@ from queue import Queue from collections import Counter -from azure.eventhub.async import EventHubClientAsync +from azure.eventhub import EventHubClientAsync from azure.eventprocessorhost.eh_partition_pump import EventHubPartitionPump from azure.eventprocessorhost.cancellation_token import CancellationToken diff --git a/examples/recv_async.py b/examples/recv_async.py index 13b4c76..d025bc9 100644 --- a/examples/recv_async.py +++ b/examples/recv_async.py @@ -14,8 +14,7 @@ import time import logging import asyncio -from azure.eventhub import Offset -from azure.eventhub.async import EventHubClientAsync, AsyncReceiver +from azure.eventhub import Offset, EventHubClientAsync, AsyncReceiver import examples logger = examples.get_logger(logging.INFO) diff --git a/examples/recv_epoch.py b/examples/recv_epoch.py index cd28751..f9f291e 100644 --- a/examples/recv_epoch.py +++ b/examples/recv_epoch.py @@ -14,8 +14,7 @@ import time import logging import asyncio -from azure.eventhub import Offset -from azure.eventhub.async import EventHubClientAsync, AsyncReceiver +from azure.eventhub import Offset, EventHubClientAsync, AsyncReceiver import examples logger = examples.get_logger(logging.INFO) diff --git a/examples/send_async.py b/examples/send_async.py index fa5e5d1..96a6ce1 100644 --- a/examples/send_async.py +++ b/examples/send_async.py @@ -12,8 +12,7 @@ import asyncio import os -from azure.eventhub import EventData -from azure.eventhub.async import EventHubClientAsync, AsyncSender +from azure.eventhub import EventData, EventHubClientAsync, AsyncSender import examples logger = examples.get_logger(logging.INFO) diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 3480374..0000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[bdist_wheel] -universal=1 \ No newline at end of file diff --git a/tests/test_negative.py b/tests/test_negative.py index 1acb8a3..2eaff40 100644 --- a/tests/test_negative.py +++ b/tests/test_negative.py @@ -9,8 +9,12 @@ import pytest from azure import eventhub -from azure.eventhub import EventData, Offset, EventHubError, EventHubClient -from azure.eventhub.async import EventHubClientAsync +from azure.eventhub import ( + EventHubClientAsync, + EventData, + Offset, + EventHubError, + EventHubClient) def test_send_partition_key_with_partition(connection_str): diff --git a/tests/test_receive_async.py b/tests/test_receive_async.py index b8347f6..68a87ef 100644 --- a/tests/test_receive_async.py +++ b/tests/test_receive_async.py @@ -10,8 +10,7 @@ import time from azure import eventhub -from azure.eventhub import EventData, Offset, EventHubError -from azure.eventhub.async import EventHubClientAsync +from azure.eventhub import EventData, Offset, EventHubError, EventHubClientAsync @pytest.mark.asyncio @@ -192,6 +191,7 @@ async def pump(receiver, sleep=None): @pytest.mark.asyncio async def test_epoch_receiver_async(connection_str, senders): + pytest.skip("") client = EventHubClientAsync.from_connection_string(connection_str, debug=False) receivers = [] for epoch in [10, 20]: @@ -237,6 +237,7 @@ async def test_multiple_receiver_async(connection_str, senders): @pytest.mark.asyncio async def test_epoch_receiver_after_non_epoch_receiver_async(connection_str, senders): + pytest.skip("") client = EventHubClientAsync.from_connection_string(connection_str, debug=False) receivers = [] receivers.append(client.add_async_receiver("$default", "0", prefetch=1000)) diff --git a/tests/test_send_async.py b/tests/test_send_async.py index af20fc7..ede8b73 100644 --- a/tests/test_send_async.py +++ b/tests/test_send_async.py @@ -9,9 +9,7 @@ import pytest import time -from azure import eventhub -from azure.eventhub import EventData -from azure.eventhub.async import EventHubClientAsync +from azure.eventhub import EventData, EventHubClientAsync @pytest.mark.asyncio From 9877eddd4f5df6ea19467207d714aaf254a8fa14 Mon Sep 17 00:00:00 2001 From: annatisch Date: Wed, 4 Jul 2018 10:52:49 -0700 Subject: [PATCH 09/52] IoThub support --- azure/eventhub/_async/__init__.py | 89 +++++++-- azure/eventhub/_async/receiver_async.py | 72 +++++-- azure/eventhub/_async/sender_async.py | 60 +++++- azure/eventhub/common.py | 13 +- azure/eventhub/receiver.py | 82 ++++++-- azure/eventhub/sender.py | 72 ++++++- .../abstract_event_processor.py | 2 +- azure/eventprocessorhost/eh_partition_pump.py | 2 +- azure/eventprocessorhost/partition_context.py | 4 +- azure/eventprocessorhost/partition_pump.py | 2 +- conftest.py | 45 ++++- examples/eph.py | 2 +- tests/__init__.py | 2 +- tests/test_iothub_receive.py | 22 +++ tests/test_iothub_receive_async.py | 66 +++++++ tests/test_iothub_send.py | 28 +++ tests/test_negative.py | 181 +++++++++++------- tests/test_receive_async.py | 42 ++-- 18 files changed, 612 insertions(+), 174 deletions(-) create mode 100644 tests/test_iothub_receive.py create mode 100644 tests/test_iothub_receive_async.py create mode 100644 tests/test_iothub_send.py diff --git a/azure/eventhub/_async/__init__.py b/azure/eventhub/_async/__init__.py index 5831051..e80d78a 100644 --- a/azure/eventhub/_async/__init__.py +++ b/azure/eventhub/_async/__init__.py @@ -8,8 +8,7 @@ import time import datetime -from uamqp import constants, types, errors -from uamqp.authentication import SASTokenAsync +from uamqp import authentication, constants, types, errors from uamqp import ( Message, Source, @@ -50,7 +49,9 @@ def _create_auth(self, auth_uri, username, password): # pylint: disable=no-self :param password: The shared access key. :type password: str """ - return SASTokenAsync.from_shared_access_key(auth_uri, username, password) + if "@sas.root" in username: + return authentication.SASLPlain(self.address.hostname, username, password) + return authentication.SASTokenAsync.from_shared_access_key(auth_uri, username, password) def _create_connection_async(self): """ @@ -79,21 +80,71 @@ async def _close_clients_async(self): """ Close all open AsyncSender/AsyncReceiver clients. """ - for client in self.clients: - await client.close_async() + await asyncio.gather(*[c.close_async() for c in self.clients]) + + async def _wait_for_client(self, client): + try: + while client.get_handler_state().value == 2: + await self.connection.work_async() + except Exception as exp: + await client.close_async(exception=exp) + + async def _start_client_async(self, client): + try: + await client.open_async(self.connection) + started = await client.has_started() + while not started: + await self.connection.work_async() + started = await client.has_started() + except Exception as exp: + await client.close_async(exception=exp) + + async def _handle_redirect(self, redirects): + if len(redirects) != len(self.clients): + not_redirected = [c for c in self.clients if not c.redirected] + done, timeout = await asyncio.wait([self._wait_for_client(c) for c in not_redirected], timeout=5) + if timeout: + raise EventHubError("Some clients are attempting to redirect the connection.") + redirects = [c.redirected for c in self.clients if c.redirected] + if not all(r.hostname == redirects[0].hostname for r in redirects): + raise EventHubError("Multiple clients attempting to redirect to different hosts.") + self.auth = self._create_auth(redirects[0].address.decode('utf-8'), **self._auth_config) + await self.connection.redirect_async(redirects[0], self.auth) + await asyncio.gather(*[c.open_async(self.connection) for c in self.clients]) async def run_async(self): """ Run the EventHubClient asynchronously. Opens the connection and starts running all AsyncSender/AsyncReceiver clients. + Returns a list of the start up results. For a succcesful client start the + result will be `None`, otherwise the exception raise. + If all clients failed to start, the run will fail, shut down the connection + and raise an exception. + If at least one client starts up successfully the run command will succeed. - :rtype: ~azure.eventhub._async.EventHubClientAsync + :rtype: list[~azure.eventhub.common.EventHubError] """ log.info("{}: Starting {} clients".format(self.container_id, len(self.clients))) self._create_connection_async() - for client in self.clients: - await client.open_async(connection=self.connection) - return self + tasks = [self._start_client_async(c) for c in self.clients] + try: + await asyncio.gather(*tasks) + redirects = [c.redirected for c in self.clients if c.redirected] + failed = [c.error for c in self.clients if c.error] + if failed and len(failed) == len(self.clients): + log.warning("{}: All clients failed to start.".format(self.container_id, len(failed))) + raise failed[0] + elif failed: + log.warning("{}: {} clients failed to start.".format(self.container_id, len(failed))) + elif redirects: + await self._handle_redirect(redirects) + except EventHubError: + await self.stop_async() + raise + except Exception as exp: + await self.stop_async() + raise EventHubError(str(exp)) + return failed async def stop_async(self): """ @@ -130,7 +181,7 @@ async def get_eventhub_info_async(self): output['partition_ids'] = [p.decode('utf-8') for p in eh_info[b'partition_ids']] return output - def add_async_receiver(self, consumer_group, partition, offset=None, prefetch=300, loop=None): + def add_async_receiver(self, consumer_group, partition, offset=None, prefetch=300, operation=None, loop=None): """ Add an async receiver to the client for a particular consumer group and partition. @@ -144,16 +195,17 @@ def add_async_receiver(self, consumer_group, partition, offset=None, prefetch=30 :type prefetch: int :rtype: ~azure.eventhub._async.receiver_async.ReceiverAsync """ + path = self.address.path + operation if operation else self.address.path source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( - self.address.hostname, self.address.path, consumer_group, partition) + self.address.hostname, path, consumer_group, partition) source = Source(source_url) if offset is not None: source.set_filter(offset.selector()) handler = AsyncReceiver(self, source, prefetch=prefetch, loop=loop) - self.clients.append(handler._handler) # pylint: disable=protected-access + self.clients.append(handler) return handler - def add_async_epoch_receiver(self, consumer_group, partition, epoch, prefetch=300, loop=None): + def add_async_epoch_receiver(self, consumer_group, partition, epoch, prefetch=300, operation=None, loop=None): """ Add an async receiver to the client with an epoch value. Only a single epoch receiver can connect to a partition at any given time - additional epoch receivers must have @@ -170,13 +222,14 @@ def add_async_epoch_receiver(self, consumer_group, partition, epoch, prefetch=30 :type prefetch: int :rtype: ~azure.eventhub._async.receiver_async.ReceiverAsync """ + path = self.address.path + operation if operation else self.address.path source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( - self.address.hostname, self.address.path, consumer_group, partition) + self.address.hostname, path, consumer_group, partition) handler = AsyncReceiver(self, source_url, prefetch=prefetch, epoch=epoch, loop=loop) - self.clients.append(handler._handler) # pylint: disable=protected-access + self.clients.append(handler) return handler - def add_async_sender(self, partition=None, loop=None): + def add_async_sender(self, partition=None, operation=None, loop=None): """ Add an async sender to the client to send ~azure.eventhub.common.EventData object to an EventHub. @@ -188,6 +241,8 @@ def add_async_sender(self, partition=None, loop=None): :rtype: ~azure.eventhub._async.sender_async.SenderAsync """ target = "amqps://{}{}".format(self.address.hostname, self.address.path) + if operation: + target = target + operation handler = AsyncSender(self, target, partition=partition, loop=loop) - self.clients.append(handler._handler) # pylint: disable=protected-access + self.clients.append(handler) return handler diff --git a/azure/eventhub/_async/receiver_async.py b/azure/eventhub/_async/receiver_async.py index 3c044d4..3506fdf 100644 --- a/azure/eventhub/_async/receiver_async.py +++ b/azure/eventhub/_async/receiver_async.py @@ -33,21 +33,65 @@ def __init__(self, client, source, prefetch=300, epoch=None, loop=None): # pyli :param loop: An event loop. """ self.loop = loop or asyncio.get_event_loop() + self.redirected = None + self.error = None + self.debug = client.debug self.offset = None self.prefetch = prefetch + self.properties = None self.epoch = epoch properties = None if epoch: - properties = {types.AMQPSymbol(self._epoch): types.AMQPLong(int(epoch))} + self.properties = {types.AMQPSymbol(self._epoch): types.AMQPLong(int(epoch))} self._handler = ReceiveClientAsync( source, auth=client.auth, - debug=client.debug, + debug=self.debug, prefetch=self.prefetch, - link_properties=properties, + link_properties=self.properties, timeout=self.timeout, loop=self.loop) + async def open_async(self, connection): + if self.redirected: + self._handler = ReceiveClientAsync( + self.redirected.address, + auth=None, + debug=self.debug, + prefetch=self.prefetch, + link_properties=self.properties, + timeout=self.timeout, + loop=self.loop) + await self._handler.open_async(connection=connection) + + async def has_started(self): + # pylint: disable=protected-access + timeout = False + auth_in_progress = False + if self._handler._connection.cbs: + timeout, auth_in_progress = await self._handler._auth.handle_token_async() + if timeout: + raise EventHubError("Authorization timeout.") + elif auth_in_progress: + return False + elif not await self._handler._client_ready(): + return False + else: + return True + + async def close_async(self, exception=None): + if self.error: + return + elif isinstance(exception, errors.LinkRedirect): + self.redirected = exception + elif isinstance(exception, EventHubError): + self.error = exception + elif exception: + self.error = EventHubError(str(exception)) + else: + self.error = EventHubError("This receive client is now closed.") + await self._handler.close_async() + async def receive(self, max_batch_size=None, timeout=None): """ Receive events asynchronously from the EventHub. @@ -58,8 +102,10 @@ async def receive(self, max_batch_size=None, timeout=None): retrieve before the time, the result will be empty. If no batch size is supplied, the prefetch size will be the maximum. :type max_batch_size: int - :rtype: list[~azure.eventhub.EventData] + :rtype: list[~azure.eventhub.common.EventData] """ + if self.error: + raise self.error try: timeout_ms = 1000 * timeout if timeout else 0 message_batch = await self._handler.receive_message_batch_async( @@ -71,15 +117,11 @@ async def receive(self, max_batch_size=None, timeout=None): self.offset = event_data.offset data_batch.append(event_data) return data_batch - except errors.AMQPConnectionError as e: - message = "Failed to open receiver: {}".format(e) - message += "\nPlease check that the partition key is valid " - if self.epoch: - message += "and that a higher epoch receiver is " \ - "not already running for this partition." - else: - message += "and whether an epoch receiver is " \ - "already running for this partition." - raise EventHubError(message) + except errors.LinkDetach as detach: + error = EventHubError(str(detach)) + await self.close_async(exception=error) + raise error except Exception as e: - raise EventHubError("Receive failed: {}".format(e)) + error = EventHubError("Receive failed: {}".format(e)) + await self.close_async(exception=error) + raise error diff --git a/azure/eventhub/_async/sender_async.py b/azure/eventhub/_async/sender_async.py index b702162..27bce63 100644 --- a/azure/eventhub/_async/sender_async.py +++ b/azure/eventhub/_async/sender_async.py @@ -5,7 +5,7 @@ import asyncio -from uamqp import constants +from uamqp import constants, errors from uamqp import SendClientAsync from azure.eventhub import EventHubError @@ -26,6 +26,9 @@ def __init__(self, client, target, partition=None, loop=None): # pylint: disabl :type target: str :param loop: An event loop. """ + self.redirected = None + self.error = None + self.debug = client.debug self.partition = partition if partition: target += "/Partitions/" + partition @@ -33,22 +36,61 @@ def __init__(self, client, target, partition=None, loop=None): # pylint: disabl self._handler = SendClientAsync( target, auth=client.auth, - debug=client.debug, + debug=self.debug, msg_timeout=Sender.TIMEOUT, loop=self.loop) self._outcome = None self._condition = None + async def open_async(self, connection): + if self.redirected: + self._handler = SendClientAsync( + self.redirected.address, + auth=None, + debug=self.debug, + msg_timeout=Sender.TIMEOUT) + await self._handler.open_async(connection=connection) + + async def has_started(self): + # pylint: disable=protected-access + timeout = False + auth_in_progress = False + if self._handler._connection.cbs: + timeout, auth_in_progress = await self._handler._auth.handle_token_async() + if timeout: + raise EventHubError("Authorization timeout.") + elif auth_in_progress: + return False + elif not await self._handler._client_ready(): + return False + else: + return True + + async def close_async(self, exception=None): + if self.error: + return + elif isinstance(exception, errors.LinkRedirect): + self.redirected = exception + elif isinstance(exception, EventHubError): + self.error = exception + elif exception: + self.error = EventHubError(str(exception)) + else: + self.error = EventHubError("This send client is now closed.") + await self._handler.close_async() + async def send(self, event_data): """ Sends an event data and asynchronously waits until acknowledgement is received or operation times out. :param event_data: The event to be sent. - :type event_data: ~azure.eventhub.EventData - :raises: ~azure.eventhub.EventHubError if the message fails to + :type event_data: ~azure.eventhub.common.EventData + :raises: ~azure.eventhub.common.EventHubError if the message fails to send. """ + if self.error: + raise self.error if event_data.partition_key and self.partition: raise ValueError("EventData partition key cannot be used with a partition sender.") event_data.message.on_send_complete = self._on_outcome @@ -56,5 +98,13 @@ async def send(self, event_data): await self._handler.send_message_async(event_data.message) if self._outcome != constants.MessageSendResult.Ok: raise Sender._error(self._outcome, self._condition) + except errors.LinkDetach as detach: + error = EventHubError(str(detach)) + await self.close_async(exception=error) + raise error except Exception as e: - raise EventHubError("Send failed: {}".format(e)) + error = EventHubError("Send failed: {}".format(e)) + await self.close_async(exception=error) + raise error + else: + return self._outcome diff --git a/azure/eventhub/common.py b/azure/eventhub/common.py index f0a7f92..f528bba 100644 --- a/azure/eventhub/common.py +++ b/azure/eventhub/common.py @@ -8,7 +8,7 @@ from uamqp import Message, BatchMessage from uamqp import types -from uamqp.message import MessageHeader +from uamqp.message import MessageHeader, MessageProperties class EventData(object): @@ -23,7 +23,7 @@ class EventData(object): PROP_TIMESTAMP = b"x-opt-enqueued-time" PROP_DEVICE_ID = b"iothub-connection-device-id" - def __init__(self, body=None, batch=None, message=None): + def __init__(self, body=None, batch=None, to_device=None, message=None): """ Initialize EventData. @@ -37,21 +37,24 @@ def __init__(self, body=None, batch=None, message=None): self._partition_key = types.AMQPSymbol(EventData.PROP_PARTITION_KEY) self._annotations = {} self._properties = {} + self._msg_properties = MessageProperties() + if to_device: + self._msg_properties.to = '/devices/{}/messages/devicebound'.format(to_device) if batch: - self.message = BatchMessage(data=batch, multi_messages=True) + self.message = BatchMessage(data=batch, multi_messages=True, properties=self._msg_properties) elif message: self.message = message self._annotations = message.annotations self._properties = message.application_properties else: if isinstance(body, list) and body: - self.message = Message(body[0]) + self.message = Message(body[0], properties=self._msg_properties) for more in body[1:]: self.message._body.append(more) # pylint: disable=protected-access elif body is None: raise ValueError("EventData cannot be None.") else: - self.message = Message(body) + self.message = Message(body, properties=self._msg_properties) @property diff --git a/azure/eventhub/receiver.py b/azure/eventhub/receiver.py index c332bca..beec103 100644 --- a/azure/eventhub/receiver.py +++ b/azure/eventhub/receiver.py @@ -21,7 +21,7 @@ def __init__(self, client, source, prefetch=300, epoch=None): Instantiate a receiver. :param client: The parent EventHubClient. - :type client: ~azure.eventhub.EventHubClient + :type client: ~azure.eventhub.client.EventHubClient :param source: The source EventHub from which to receive events. :type source: ~uamqp.address.Source :param prefetch: The number of events to prefetch from the service @@ -33,17 +33,63 @@ def __init__(self, client, source, prefetch=300, epoch=None): self.offset = None self.prefetch = prefetch self.epoch = epoch - properties = None + self.properties = None + self.redirected = None + self.debug = client.debug + self.error = None if epoch: - properties = {types.AMQPSymbol(self._epoch): types.AMQPLong(int(epoch))} + self.properties = {types.AMQPSymbol(self._epoch): types.AMQPLong(int(epoch))} self._handler = ReceiveClient( source, auth=client.auth, - debug=client.debug, + debug=self.debug, prefetch=self.prefetch, - link_properties=properties, + link_properties=self.properties, timeout=self.timeout) + def open(self, connection): + if self.redirected: + self._handler = ReceiveClient( + self.redirected.address, + auth=None, + debug=self.debug, + prefetch=self.prefetch, + link_properties=self.properties, + timeout=self.timeout) + self._handler.open(connection) + + def get_handler_state(self): + # pylint: disable=protected-access + return self._handler._message_receiver.get_state() + + def has_started(self): + # pylint: disable=protected-access + timeout = False + auth_in_progress = False + if self._handler._connection.cbs: + timeout, auth_in_progress = self._handler._auth.handle_token() + if timeout: + raise EventHubError("Authorization timeout.") + elif auth_in_progress: + return False + elif not self._handler._client_ready(): + return False + else: + return True + + def close(self, exception=None): + if self.error: + return + elif isinstance(exception, errors.LinkRedirect): + self.redirected = exception + elif isinstance(exception, EventHubError): + self.error = exception + elif exception: + self.error = EventHubError(str(exception)) + else: + self.error = EventHubError("This receive client is now closed.") + self._handler.close() + @property def queue_size(self): """ @@ -66,8 +112,10 @@ def receive(self, max_batch_size=None, timeout=None): retrieve before the time, the result will be empty. If no batch size is supplied, the prefetch size will be the maximum. :type max_batch_size: int - :rtype: list[~azure.eventhub.EventData] + :rtype: list[~azure.eventhub.common.EventData] """ + if self.error: + raise self.error try: timeout_ms = 1000 * timeout if timeout else 0 message_batch = self._handler.receive_message_batch( @@ -79,26 +127,22 @@ def receive(self, max_batch_size=None, timeout=None): self.offset = event_data.offset data_batch.append(event_data) return data_batch - except errors.AMQPConnectionError as e: - message = "Failed to open receiver: {}".format(e) - message += "\nPlease check that the partition key is valid " - if self.epoch: - message += ("and that a higher epoch receiver is not " - "already running for this partition.") - else: - message += ("and whether an epoch receiver is " - "already running for this partition.") - raise EventHubError(message) + except errors.LinkDetach as detach: + error = EventHubError(str(detach)) + self.close(exception=error) + raise error except Exception as e: - raise EventHubError("Receive failed: {}".format(e)) + error = EventHubError("Receive failed: {}".format(e)) + self.close(exception=error) + raise error def selector(self, default): """ Create a selector for the current offset if it is set. :param default: The fallback receive offset. - :type default: ~azure.eventhub.Offset - :rtype: ~azure.eventhub.Offset + :type default: ~azure.eventhub.common.Offset + :rtype: ~azure.eventhub.common.Offset """ if self.offset is not None: return Offset(self.offset).selector() diff --git a/azure/eventhub/sender.py b/azure/eventhub/sender.py index 78c2195..0a0e08c 100644 --- a/azure/eventhub/sender.py +++ b/azure/eventhub/sender.py @@ -3,7 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -from uamqp import constants +from uamqp import constants, errors from uamqp import SendClient from azure.eventhub.common import EventHubError @@ -20,33 +20,79 @@ def __init__(self, client, target, partition=None): Instantiate an EventHub event Sender client. :param client: The parent EventHubClient. - :type client: ~azure.eventhub.EventHubClient. + :type client: ~azure.eventhub.client.EventHubClient. :param target: The URI of the EventHub to send to. :type target: str """ + self.redirected = None + self.error = None + self.debug = client.debug self.partition = partition if partition: target += "/Partitions/" + partition self._handler = SendClient( target, auth=client.auth, - debug=client.debug, + debug=self.debug, msg_timeout=Sender.TIMEOUT) self._outcome = None self._condition = None + def open(self, connection): + if self.redirected: + self._handler = SendClient( + self.redirected.address, + auth=None, + debug=self.debug, + msg_timeout=Sender.TIMEOUT) + self._handler.open(connection) + + def get_handler_state(self): + # pylint: disable=protected-access + return self._handler._message_sender.get_state() + + def has_started(self): + # pylint: disable=protected-access + timeout = False + auth_in_progress = False + if self._handler._connection.cbs: + timeout, auth_in_progress = self._handler._auth.handle_token() + if timeout: + raise EventHubError("Authorization timeout.") + elif auth_in_progress: + return False + elif not self._handler._client_ready(): + return False + else: + return True + + def close(self, exception=None): + if self.error: + return + elif isinstance(exception, errors.LinkRedirect): + self.redirected = exception + elif isinstance(exception, EventHubError): + self.error = exception + elif exception: + self.error = EventHubError(str(exception)) + else: + self.error = EventHubError("This send client is now closed.") + self._handler.close() + def send(self, event_data): """ Sends an event data and blocks until acknowledgement is received or operation times out. :param event_data: The event to be sent. - :type event_data: ~azure.eventhub.client.EventData - :raises: ~azure.eventhub.client.EventHubError if the message fails to + :type event_data: ~azure.eventhub.common.EventData + :raises: ~azure.eventhub.common.EventHubError if the message fails to send. :return: The outcome of the message send. :rtype: ~uamqp.constants.MessageSendResult """ + if self.error: + raise self.error if event_data.partition_key and self.partition: raise ValueError("EventData partition key cannot be used with a partition sender.") event_data.message.on_send_complete = self._on_outcome @@ -54,8 +100,14 @@ def send(self, event_data): self._handler.send_message(event_data.message) if self._outcome != constants.MessageSendResult.Ok: raise Sender._error(self._outcome, self._condition) + except errors.LinkDetach as detach: + error = EventHubError(str(detach)) + self.close(exception=error) + raise error except Exception as e: - raise EventHubError("Send failed: {}".format(e)) + error = EventHubError("Send failed: {}".format(e)) + self.close(exception=error) + raise error else: return self._outcome @@ -64,11 +116,13 @@ def transfer(self, event_data, callback=None): Transfers an event data and notifies the callback when the operation is done. :param event_data: The event to be sent. - :type event_data: ~azure.eventhub.client.EventData + :type event_data: ~azure.eventhub.common.EventData :param callback: Callback to be run once the message has been send. This must be a function that accepts two arguments. - :type callback: func[~uamqp.constants.MessageSendResult, ~azure.eventhub.client.EventHubError] + :type callback: func[~uamqp.constants.MessageSendResult, ~azure.eventhub.common.EventHubError] """ + if self.error: + raise self.error if event_data.partition_key and self.partition: raise ValueError("EventData partition key cannot be used with a partition sender.") if callback: @@ -79,6 +133,8 @@ def wait(self): """ Wait until all transferred events have been sent. """ + if self.error: + raise self.error try: self._handler.wait() except Exception as e: diff --git a/azure/eventprocessorhost/abstract_event_processor.py b/azure/eventprocessorhost/abstract_event_processor.py index 6ff9dd4..12302b5 100644 --- a/azure/eventprocessorhost/abstract_event_processor.py +++ b/azure/eventprocessorhost/abstract_event_processor.py @@ -43,7 +43,7 @@ async def process_events_async(self, context, messages): :param context: Information about the partition :type context: ~azure.eventprocessorhost.partition_context.PartitionContext :param messages: The events to be processed. - :type messages: list[~azure.eventhub.EventData] + :type messages: list[~azure.eventhub.common.EventData] """ pass diff --git a/azure/eventprocessorhost/eh_partition_pump.py b/azure/eventprocessorhost/eh_partition_pump.py index 1c1b813..86c42d2 100644 --- a/azure/eventprocessorhost/eh_partition_pump.py +++ b/azure/eventprocessorhost/eh_partition_pump.py @@ -141,7 +141,7 @@ async def process_events_async(self, events): by running faster than OnEvents. :param events: List of events to be processed. - :type events: list of ~azure.eventhub.EventData + :type events: list of ~azure.eventhub.common.EventData """ await self.eh_partition_pump.process_events_async(events) diff --git a/azure/eventprocessorhost/partition_context.py b/azure/eventprocessorhost/partition_context.py index fb619b3..9eaf53f 100644 --- a/azure/eventprocessorhost/partition_context.py +++ b/azure/eventprocessorhost/partition_context.py @@ -30,7 +30,7 @@ def set_offset_and_sequence_number(self, event_data): Updates offset based on event. :param event_data: A received EventData with valid offset and sequenceNumber. - :type event_data: ~azure.eventhub.EventData + :type event_data: ~azure.eventhub.common.EventData """ if not event_data: raise Exception(event_data) @@ -73,7 +73,7 @@ async def checkpoint_async_event_data(self, event_data): then writes those values to the checkpoint store via the checkpoint manager. :param event_data: A received EventData with valid offset and sequenceNumber. - :type event_data: ~azure.eventhub.EventData + :type event_data: ~azure.eventhub.common.EventData :raises: ValueError if suplied event_data is None. :raises: ValueError if the sequenceNumber is less than the last checkpointed value. """ diff --git a/azure/eventprocessorhost/partition_pump.py b/azure/eventprocessorhost/partition_pump.py index 26e62b0..769e2ce 100644 --- a/azure/eventprocessorhost/partition_pump.py +++ b/azure/eventprocessorhost/partition_pump.py @@ -136,7 +136,7 @@ async def process_events_async(self, events): Process pump events. :param events: List of events to be processed. - :type events: list[~azure.eventhub.EventData] + :type events: list[~azure.eventhub.common.EventData] """ if events: # Synchronize to serialize calls to the processor. The handler is not installed until diff --git a/conftest.py b/conftest.py index 1673946..6932e27 100644 --- a/conftest.py +++ b/conftest.py @@ -23,7 +23,7 @@ from azure.eventprocessorhost.abstract_event_processor import AbstractEventProcessor -log = get_logger(None, logging.INFO) +log = get_logger(None, logging.DEBUG) @pytest.fixture() def live_eventhub_config(): @@ -49,6 +49,49 @@ def connection_str(): pytest.skip("No EventHub connection string found.") +@pytest.fixture() +def invalid_hostname(): + try: + conn_str = os.environ['EVENT_HUB_CONNECTION_STR'] + return conn_str.replace("Endpoint=sb://", "Endpoint=sb://invalid.") + except KeyError: + pytest.skip("No EventHub connection string found.") + + +@pytest.fixture() +def invalid_key(): + try: + conn_str = os.environ['EVENT_HUB_CONNECTION_STR'] + return conn_str.replace("SharedAccessKey=", "SharedAccessKey=invalid") + except KeyError: + pytest.skip("No EventHub connection string found.") + + +@pytest.fixture() +def invalid_policy(): + try: + conn_str = os.environ['EVENT_HUB_CONNECTION_STR'] + return conn_str.replace("SharedAccessKeyName=", "SharedAccessKeyName=invalid") + except KeyError: + pytest.skip("No EventHub connection string found.") + + +@pytest.fixture() +def iot_connection_str(): + try: + return os.environ['IOT_HUB_CONNECTION_STR'] + except KeyError: + pytest.skip("No IotHub connection string found.") + + +@pytest.fixture() +def device_id(): + try: + return os.environ['IOTHUB_DEVICE'] + except KeyError: + pytest.skip("No Iothub device ID found.") + + @pytest.fixture() def receivers(connection_str): client = EventHubClient.from_connection_string(connection_str, debug=True) diff --git a/examples/eph.py b/examples/eph.py index 9ad16e7..7a9633c 100644 --- a/examples/eph.py +++ b/examples/eph.py @@ -58,7 +58,7 @@ async def process_events_async(self, context, messages): :param context: Information about the partition :type context: ~azure.eventprocessorhost.PartitionContext :param messages: The events to be processed. - :type messages: list[~azure.eventhub.EventData] + :type messages: list[~azure.eventhub.common.EventData] """ logger.info("Events processed {}".format(context.sequence_number)) await context.checkpoint_async() diff --git a/tests/__init__.py b/tests/__init__.py index 7b7c91a..7ec7d3b 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -12,7 +12,7 @@ def get_logger(filename, level=logging.INFO): azure_logger = logging.getLogger("azure") azure_logger.setLevel(level) uamqp_logger = logging.getLogger("uamqp") - uamqp_logger.setLevel(logging.INFO) + uamqp_logger.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s') console_handler = logging.StreamHandler(stream=sys.stdout) diff --git a/tests/test_iothub_receive.py b/tests/test_iothub_receive.py new file mode 100644 index 0000000..78c1de8 --- /dev/null +++ b/tests/test_iothub_receive.py @@ -0,0 +1,22 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +#-------------------------------------------------------------------------- + +import os +import pytest +import time + +from azure import eventhub +from azure.eventhub import EventData, EventHubClient, Offset + +def test_iothub_receive(iot_connection_str, device_id): + client = EventHubClient.from_iothub_connection_string(iot_connection_str, debug=True) + receiver = client.add_receiver("$default", "0", operation='/messages/events') + try: + client.run() + received = receiver.receive(timeout=5) + assert len(received) == 0 + finally: + client.stop() \ No newline at end of file diff --git a/tests/test_iothub_receive_async.py b/tests/test_iothub_receive_async.py new file mode 100644 index 0000000..a7126d3 --- /dev/null +++ b/tests/test_iothub_receive_async.py @@ -0,0 +1,66 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +#-------------------------------------------------------------------------- + +import os +import asyncio +import pytest +import time + +from azure import eventhub +from azure.eventhub import EventData, Offset, EventHubError, EventHubClientAsync + + +async def pump(receiver, sleep=None): + messages = 0 + if sleep: + await asyncio.sleep(sleep) + batch = await receiver.receive(timeout=1) + while batch: + messages += len(batch) + batch = await receiver.receive(timeout=1) + return messages + + +@pytest.mark.asyncio +async def test_iothub_receive_async(iot_connection_str): + client = EventHubClientAsync.from_iothub_connection_string(iot_connection_str, debug=True) + receivers = [] + for i in range(2): + receivers.append(client.add_async_receiver("$default", "0", prefetch=1000, operation='/messages/events')) + await client.run_async() + try: + outputs = await asyncio.gather( + pump(receivers[0]), + pump(receivers[1]), + return_exceptions=True) + + assert isinstance(outputs[0], int) and outputs[0] == 0 + assert isinstance(outputs[1], int) and outputs[1] == 0 + except: + raise + finally: + await client.stop_async() + + +@pytest.mark.asyncio +async def test_iothub_receive_detach_async(iot_connection_str): + client = EventHubClientAsync.from_iothub_connection_string(iot_connection_str, debug=True) + receivers = [] + for i in range(2): + receivers.append(client.add_async_receiver("$default", str(i), prefetch=1000, operation='/messages/events')) + await client.run_async() + try: + outputs = await asyncio.gather( + pump(receivers[0]), + pump(receivers[1]), + return_exceptions=True) + + assert isinstance(outputs[0], int) and outputs[0] == 0 + assert isinstance(outputs[1], EventHubError) + except: + raise + finally: + await client.stop_async() \ No newline at end of file diff --git a/tests/test_iothub_send.py b/tests/test_iothub_send.py new file mode 100644 index 0000000..7c0dd7c --- /dev/null +++ b/tests/test_iothub_send.py @@ -0,0 +1,28 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +#-------------------------------------------------------------------------- + +import os +import pytest +import time +import uuid + +from uamqp.message import MessageProperties + +from azure import eventhub +from azure.eventhub import EventData, EventHubClient + + +def test_iothub_send_single_event(iot_connection_str, device_id): + client = EventHubClient.from_iothub_connection_string(iot_connection_str, debug=True) + sender = client.add_sender(operation='/messages/devicebound') + try: + client.run() + outcome = sender.send(EventData(b"A single event", to_device=device_id)) + assert outcome.value == 0 + except: + raise + finally: + client.stop() diff --git a/tests/test_negative.py b/tests/test_negative.py index 2eaff40..3c8dde6 100644 --- a/tests/test_negative.py +++ b/tests/test_negative.py @@ -17,8 +17,98 @@ EventHubClient) -def test_send_partition_key_with_partition(connection_str): - client = EventHubClient.from_connection_string(connection_str, debug=False) +def test_send_with_invalid_hostname(invalid_hostname, receivers): + client = EventHubClient.from_connection_string(invalid_hostname, debug=False) + sender = client.add_sender() + with pytest.raises(EventHubError): + client.run() + + +@pytest.mark.asyncio +async def test_send_with_invalid_hostname_async(invalid_hostname, receivers): + client = EventHubClientAsync.from_connection_string(invalid_hostname, debug=True) + sender = client.add_async_sender() + with pytest.raises(EventHubError): + await client.run_async() + + +def test_receive_with_invalid_hostname_sync(invalid_hostname): + client = EventHubClient.from_connection_string(invalid_hostname, debug=True) + receiver = client.add_receiver("$default", "0") + with pytest.raises(EventHubError): + client.run() + + +@pytest.mark.asyncio +async def test_receive_with_invalid_hostname_async(invalid_hostname): + client = EventHubClientAsync.from_connection_string(invalid_hostname, debug=True) + sender = client.add_async_receiver("$default", "0") + with pytest.raises(EventHubError): + await client.run_async() + + +def test_send_with_invalid_key(invalid_key, receivers): + client = EventHubClient.from_connection_string(invalid_key, debug=False) + sender = client.add_sender() + with pytest.raises(EventHubError): + client.run() + + +@pytest.mark.asyncio +async def test_send_with_invalid_key_async(invalid_key, receivers): + client = EventHubClientAsync.from_connection_string(invalid_key, debug=False) + sender = client.add_async_sender() + with pytest.raises(EventHubError): + await client.run_async() + + +def test_receive_with_invalid_key_sync(invalid_key): + client = EventHubClient.from_connection_string(invalid_key, debug=True) + receiver = client.add_receiver("$default", "0") + with pytest.raises(EventHubError): + client.run() + + +@pytest.mark.asyncio +async def test_receive_with_invalid_key_async(invalid_key): + client = EventHubClientAsync.from_connection_string(invalid_key, debug=True) + sender = client.add_async_receiver("$default", "0") + with pytest.raises(EventHubError): + await client.run_async() + + +def test_send_with_invalid_policy(invalid_policy, receivers): + client = EventHubClient.from_connection_string(invalid_policy, debug=False) + sender = client.add_sender() + with pytest.raises(EventHubError): + client.run() + + +@pytest.mark.asyncio +async def test_send_with_invalid_policy_async(invalid_policy, receivers): + client = EventHubClientAsync.from_connection_string(invalid_policy, debug=False) + sender = client.add_async_sender() + with pytest.raises(EventHubError): + await client.run_async() + + +def test_receive_with_invalid_policy_sync(invalid_policy): + client = EventHubClient.from_connection_string(invalid_policy, debug=True) + receiver = client.add_receiver("$default", "0") + with pytest.raises(EventHubError): + client.run() + + +@pytest.mark.asyncio +async def test_receive_with_invalid_policy_async(invalid_policy): + client = EventHubClientAsync.from_connection_string(invalid_policy, debug=True) + sender = client.add_async_receiver("$default", "0") + with pytest.raises(EventHubError): + await client.run_async() + + +def test_send_partition_key_with_partition_sync(connection_str): + client = EventHubClient.from_connection_string(connection_str, debug=True) sender = client.add_sender(partition="1") try: client.run() @@ -26,15 +116,13 @@ def test_send_partition_key_with_partition(connection_str): data.partition_key = b"PKey" with pytest.raises(ValueError): sender.send(data) - except: - raise finally: client.stop() @pytest.mark.asyncio async def test_send_partition_key_with_partition_async(connection_str): - client = EventHubClientAsync.from_connection_string(connection_str, debug=False) + client = EventHubClientAsync.from_connection_string(connection_str, debug=True) sender = client.add_async_sender(partition="1") try: await client.run_async() @@ -42,8 +130,6 @@ async def test_send_partition_key_with_partition_async(connection_str): data.partition_key = b"PKey" with pytest.raises(ValueError): await sender.send(data) - except: - raise finally: await client.stop_async() @@ -51,70 +137,42 @@ async def test_send_partition_key_with_partition_async(connection_str): def test_non_existing_entity_sender(connection_str): client = EventHubClient.from_connection_string(connection_str, eventhub="nemo", debug=False) sender = client.add_sender(partition="1") - try: + with pytest.raises(EventHubError): client.run() - data = EventData(b"Data") - with pytest.raises(EventHubError): - sender.send(data) - except: - raise - finally: - client.stop() @pytest.mark.asyncio async def test_non_existing_entity_sender_async(connection_str): client = EventHubClientAsync.from_connection_string(connection_str, eventhub="nemo", debug=False) sender = client.add_async_sender(partition="1") - try: + with pytest.raises(EventHubError): await client.run_async() - data = EventData(b"Data") - with pytest.raises(EventHubError): - await sender.send(data) - except: - raise - finally: - await client.stop_async() def test_non_existing_entity_receiver(connection_str): client = EventHubClient.from_connection_string(connection_str, eventhub="nemo", debug=False) receiver = client.add_receiver("$default", "0") - try: + with pytest.raises(EventHubError): client.run() - with pytest.raises(EventHubError): - receiver.receive(timeout=5) - except: - raise - finally: - client.stop() @pytest.mark.asyncio async def test_non_existing_entity_receiver_async(connection_str): client = EventHubClientAsync.from_connection_string(connection_str, eventhub="nemo", debug=False) receiver = client.add_async_receiver("$default", "0") - try: + with pytest.raises(EventHubError): await client.run_async() - with pytest.raises(EventHubError): - await receiver.receive(timeout=5) - except: - raise - finally: - await client.stop_async() -def test_receive_from_invalid_partitions(connection_str): +def test_receive_from_invalid_partitions_sync(connection_str): partitions = ["XYZ", "-1", "1000", "-" ] for p in partitions: - client = EventHubClient.from_connection_string(connection_str, debug=False) + client = EventHubClient.from_connection_string(connection_str, debug=True) receiver = client.add_receiver("$default", p) try: - client.run() with pytest.raises(EventHubError): - receiver.receive(timeout=5) - except: - raise + client.run() + receiver.receive(timeout=10) finally: client.stop() @@ -126,11 +184,9 @@ async def test_receive_from_invalid_partitions_async(connection_str): client = EventHubClientAsync.from_connection_string(connection_str, debug=False) receiver = client.add_async_receiver("$default", p) try: - await client.run_async() with pytest.raises(EventHubError): - await receiver.receive(timeout=5) - except: - raise + await client.run_async() + await receiver.receive(timeout=10) finally: await client.stop_async() @@ -140,13 +196,11 @@ def test_send_to_invalid_partitions(connection_str): for p in partitions: client = EventHubClient.from_connection_string(connection_str, debug=False) sender = client.add_sender(partition=p) + client.run() + data = EventData(b"A" * 300000) try: - client.run() - data = EventData(b"Data") with pytest.raises(EventHubError): sender.send(data) - except: - raise finally: client.stop() @@ -157,19 +211,16 @@ async def test_send_to_invalid_partitions_async(connection_str): for p in partitions: client = EventHubClientAsync.from_connection_string(connection_str, debug=False) sender = client.add_async_sender(partition=p) + await client.run_async() + data = EventData(b"A" * 300000) try: - await client.run_async() - data = EventData(b"Data") with pytest.raises(EventHubError): await sender.send(data) - except: - raise finally: await client.stop_async() def test_send_too_large_message(connection_str): - partitions = ["XYZ", "-1", "1000", "-" ] client = EventHubClient.from_connection_string(connection_str, debug=False) sender = client.add_sender() try: @@ -177,15 +228,12 @@ def test_send_too_large_message(connection_str): data = EventData(b"A" * 300000) with pytest.raises(EventHubError): sender.send(data) - except: - raise finally: client.stop() @pytest.mark.asyncio async def test_send_too_large_message_async(connection_str): - partitions = ["XYZ", "-1", "1000", "-" ] client = EventHubClientAsync.from_connection_string(connection_str, debug=False) sender = client.add_async_sender() try: @@ -193,8 +241,6 @@ async def test_send_too_large_message_async(connection_str): data = EventData(b"A" * 300000) with pytest.raises(EventHubError): await sender.send(data) - except: - raise finally: await client.stop_async() @@ -208,15 +254,12 @@ def test_send_null_body(connection_str): with pytest.raises(ValueError): data = EventData(None) sender.send(data) - except: - raise finally: client.stop() @pytest.mark.asyncio async def test_send_null_body_async(connection_str): - partitions = ["XYZ", "-1", "1000", "-" ] client = EventHubClientAsync.from_connection_string(connection_str, debug=False) sender = client.add_async_sender() try: @@ -224,8 +267,6 @@ async def test_send_null_body_async(connection_str): with pytest.raises(ValueError): data = EventData(None) await sender.send(data) - except: - raise finally: await client.stop_async() @@ -241,12 +282,12 @@ async def pump(receiver): @pytest.mark.asyncio async def test_max_receivers_async(connection_str, senders): - client = EventHubClientAsync.from_connection_string(connection_str, debug=False) + client = EventHubClientAsync.from_connection_string(connection_str, debug=True) receivers = [] for i in range(6): receivers.append(client.add_async_receiver("$default", "0", prefetch=1000, offset=Offset('@latest'))) - await client.run_async() try: + await client.run_async() outputs = await asyncio.gather( pump(receivers[0]), pump(receivers[1]), @@ -255,9 +296,7 @@ async def test_max_receivers_async(connection_str, senders): pump(receivers[4]), pump(receivers[5]), return_exceptions=True) + print(outputs) assert len([o for o in outputs if isinstance(o, EventHubError)]) == 1 - - except: - raise finally: await client.stop_async() \ No newline at end of file diff --git a/tests/test_receive_async.py b/tests/test_receive_async.py index 68a87ef..d002674 100644 --- a/tests/test_receive_async.py +++ b/tests/test_receive_async.py @@ -180,10 +180,14 @@ async def test_receive_batch_async(connection_str, senders): async def pump(receiver, sleep=None): messages = 0 + count = 0 if sleep: await asyncio.sleep(sleep) batch = await receiver.receive(timeout=10) while batch: + count += 1 + if count >= 10: + break messages += len(batch) batch = await receiver.receive(timeout=10) return messages @@ -191,21 +195,17 @@ async def pump(receiver, sleep=None): @pytest.mark.asyncio async def test_epoch_receiver_async(connection_str, senders): - pytest.skip("") client = EventHubClientAsync.from_connection_string(connection_str, debug=False) receivers = [] for epoch in [10, 20]: - receivers.append(client.add_async_epoch_receiver("$default", "0", epoch, prefetch=1000)) - await client.run_async() + receivers.append(client.add_async_epoch_receiver("$default", "0", epoch, prefetch=5)) try: + await client.run_async() outputs = await asyncio.gather( pump(receivers[0]), pump(receivers[1]), return_exceptions=True) - # Depending on how many messages are present and how long the test - # runs, one receiver may not throw and error - in which case it should - # still not have received any messages. - assert isinstance(outputs[0], EventHubError) or outputs[0] == 0 + assert isinstance(outputs[0], EventHubError) assert outputs[1] >= 1 except: raise @@ -215,18 +215,16 @@ async def test_epoch_receiver_async(connection_str, senders): @pytest.mark.asyncio async def test_multiple_receiver_async(connection_str, senders): - pytest.skip("") client = EventHubClientAsync.from_connection_string(connection_str, debug=True) receivers = [] for i in range(2): - receivers.append(client.add_async_receiver("$default", "0", prefetch=1000)) - await client.run_async() + receivers.append(client.add_async_receiver("$default", "0", prefetch=10)) try: + await client.run_async() outputs = await asyncio.gather( pump(receivers[0]), pump(receivers[1]), return_exceptions=True) - print(outputs) assert isinstance(outputs[0], int) and outputs[0] >= 1 assert isinstance(outputs[1], int) and outputs[1] >= 1 except: @@ -237,23 +235,17 @@ async def test_multiple_receiver_async(connection_str, senders): @pytest.mark.asyncio async def test_epoch_receiver_after_non_epoch_receiver_async(connection_str, senders): - pytest.skip("") client = EventHubClientAsync.from_connection_string(connection_str, debug=False) receivers = [] - receivers.append(client.add_async_receiver("$default", "0", prefetch=1000)) - receivers.append(client.add_async_epoch_receiver("$default", "0", 15, prefetch=1000)) - - await client.run_async() + receivers.append(client.add_async_receiver("$default", "0", prefetch=10)) + receivers.append(client.add_async_epoch_receiver("$default", "0", 15, prefetch=10)) try: + await client.run_async() outputs = await asyncio.gather( pump(receivers[0]), pump(receivers[1], sleep=5), return_exceptions=True) - # Depending on how many messages are present and how long the test - # runs, one receiver may not throw and error - in which case it should - # still not have received any messages. - print(outputs) - assert isinstance(outputs[0], EventHubError) or outputs[0] == 0 + assert isinstance(outputs[0], EventHubError) assert isinstance(outputs[1], int) and outputs[1] >= 1 except: raise @@ -265,16 +257,14 @@ async def test_epoch_receiver_after_non_epoch_receiver_async(connection_str, sen async def test_non_epoch_receiver_after_epoch_receiver_async(connection_str, senders): client = EventHubClientAsync.from_connection_string(connection_str, debug=False) receivers = [] - receivers.append(client.add_async_epoch_receiver("$default", "0", 15, prefetch=1000)) - receivers.append(client.add_async_receiver("$default", "0", prefetch=1000)) - - await client.run_async() + receivers.append(client.add_async_epoch_receiver("$default", "0", 15, prefetch=10)) + receivers.append(client.add_async_receiver("$default", "0", prefetch=10)) try: + await client.run_async() outputs = await asyncio.gather( pump(receivers[0]), pump(receivers[1]), return_exceptions=True) - print(outputs) assert isinstance(outputs[1], EventHubError) assert isinstance(outputs[0], int) and outputs[0] >= 1 except: From 70a07a322c2854ae2949fa5af382a6b20a58e484 Mon Sep 17 00:00:00 2001 From: annatisch Date: Thu, 5 Jul 2018 12:05:18 -0700 Subject: [PATCH 10/52] Updates for RC1 release --- HISTORY.rst | 6 +- azure/eventhub/__init__.py | 2 +- azure/eventhub/_async/__init__.py | 8 +-- azure/eventhub/_async/receiver_async.py | 1 - azure/eventhub/client.py | 82 +++++++++++++++++++------ azure/eventprocessorhost/eph.py | 3 +- pylintrc | 2 +- setup.py | 2 +- tests/__init__.py | 2 +- tests/test_longrunning_receive.py | 7 +-- 10 files changed, 80 insertions(+), 35 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 3d567d5..9e63944 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,11 +8,11 @@ Release History - **Breaking change** Restructured library to support Python 3.7. Submodule `async` has been renamed and all classes from this module can now be imported from azure.eventhub directly. -- Updated uAMQP dependency to vRC2 -- Added support for constructing IoTHub connections. -- Removed optional `callback` argument from `Receiver.receive` and `AsyncReceiver.receive`. +- **Breaking change** Removed optional `callback` argument from `Receiver.receive` and `AsyncReceiver.receive`. This removes the potential for messages to be processed via callback for not yet returned in the batch. +- Updated uAMQP dependency to v0.1.0 +- Added support for constructing IoTHub connections. - Fixed memory leak in receive operations. - Dropped Python 2.7 wheel support. diff --git a/azure/eventhub/__init__.py b/azure/eventhub/__init__.py index d730dcb..5182b38 100644 --- a/azure/eventhub/__init__.py +++ b/azure/eventhub/__init__.py @@ -3,7 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -__version__ = "0.2.0b2" +__version__ = "0.2.0rc1" from azure.eventhub.common import EventData, EventHubError, Offset from azure.eventhub.client import EventHubClient diff --git a/azure/eventhub/_async/__init__.py b/azure/eventhub/_async/__init__.py index e80d78a..b17394c 100644 --- a/azure/eventhub/_async/__init__.py +++ b/azure/eventhub/_async/__init__.py @@ -86,7 +86,7 @@ async def _wait_for_client(self, client): try: while client.get_handler_state().value == 2: await self.connection.work_async() - except Exception as exp: + except Exception as exp: # pylint: disable=broad-except await client.close_async(exception=exp) async def _start_client_async(self, client): @@ -96,13 +96,13 @@ async def _start_client_async(self, client): while not started: await self.connection.work_async() started = await client.has_started() - except Exception as exp: + except Exception as exp: # pylint: disable=broad-except await client.close_async(exception=exp) async def _handle_redirect(self, redirects): if len(redirects) != len(self.clients): not_redirected = [c for c in self.clients if not c.redirected] - done, timeout = await asyncio.wait([self._wait_for_client(c) for c in not_redirected], timeout=5) + _, timeout = await asyncio.wait([self._wait_for_client(c) for c in not_redirected], timeout=5) if timeout: raise EventHubError("Some clients are attempting to redirect the connection.") redirects = [c.redirected for c in self.clients if c.redirected] @@ -132,7 +132,7 @@ async def run_async(self): redirects = [c.redirected for c in self.clients if c.redirected] failed = [c.error for c in self.clients if c.error] if failed and len(failed) == len(self.clients): - log.warning("{}: All clients failed to start.".format(self.container_id, len(failed))) + log.warning("{}: All clients failed to start.".format(self.container_id)) raise failed[0] elif failed: log.warning("{}: {} clients failed to start.".format(self.container_id, len(failed))) diff --git a/azure/eventhub/_async/receiver_async.py b/azure/eventhub/_async/receiver_async.py index 3506fdf..9438fae 100644 --- a/azure/eventhub/_async/receiver_async.py +++ b/azure/eventhub/_async/receiver_async.py @@ -40,7 +40,6 @@ def __init__(self, client, source, prefetch=300, epoch=None, loop=None): # pyli self.prefetch = prefetch self.properties = None self.epoch = epoch - properties = None if epoch: self.properties = {types.AMQPSymbol(self._epoch): types.AMQPLong(int(epoch))} self._handler = ReceiveClientAsync( diff --git a/azure/eventhub/client.py b/azure/eventhub/client.py index d0a6687..91bc482 100644 --- a/azure/eventhub/client.py +++ b/azure/eventhub/client.py @@ -23,6 +23,7 @@ from azure.eventhub import __version__ from azure.eventhub.sender import Sender from azure.eventhub.receiver import Receiver +from azure.eventhub.common import EventHubError log = logging.getLogger(__name__) @@ -36,6 +37,8 @@ def _parse_conn_str(conn_str): key, _, value = element.partition('=') if key.lower() == 'endpoint': endpoint = value.rstrip('/') + elif key.lower() == 'hostname': + endpoint = value.rstrip('/') elif key.lower() == 'sharedaccesskeyname': shared_access_key_name = value elif key.lower() == 'sharedaccesskey': @@ -113,6 +116,7 @@ def __init__(self, address, username=None, password=None, debug=False): raise ValueError("Missing username and/or password.") auth_uri = "sb://{}{}".format(self.address.hostname, self.address.path) self.auth = self._create_auth(auth_uri, username, password) + self._auth_config = None self.connection = None self.debug = debug @@ -141,7 +145,9 @@ def from_iothub_connection_string(cls, conn_str, **kwargs): hub_name = address.split('.')[0] username = "{}@sas.root.{}".format(policy, hub_name) password = _generate_sas_token(address, policy, key) - return cls(address, username=username, password=password, **kwargs) + client = cls("amqps://" + address, username=username, password=password, **kwargs) + client._auth_config = {'username': policy, 'password': key} # pylint: disable=protected-access + return client def _create_auth(self, auth_uri, username, password): # pylint: disable=no-self-use """ @@ -155,6 +161,8 @@ def _create_auth(self, auth_uri, username, password): # pylint: disable=no-self :param password: The shared access key. :type password: str """ + if "@sas.root" in username: + return authentication.SASLPlain(self.address.hostname, username, password) return authentication.SASTokenAuth.from_shared_access_key(auth_uri, username, password) def _create_properties(self): # pylint: disable=no-self-use @@ -201,18 +209,52 @@ def _close_clients(self): for client in self.clients: client.close() + def _start_clients(self): + for client in self.clients: + try: + client.open(self.connection) + while not client.has_started(): + self.connection.work() + except Exception as exp: # pylint: disable=broad-except + client.close(exception=exp) + + def _handle_redirect(self, redirects): + if len(redirects) != len(self.clients): + raise EventHubError("Some clients are attempting to redirect the connection.") + if not all(r.hostname == redirects[0].hostname for r in redirects): + raise EventHubError("Multiple clients attempting to redirect to different hosts.") + self.auth = self._create_auth(redirects[0].address.decode('utf-8'), **self._auth_config) + self.connection.redirect(redirects[0], self.auth) + for client in self.clients: + client.open(self.connection) + def run(self): """ Run the EventHubClient in blocking mode. Opens the connection and starts running all Sender/Receiver clients. - :rtype: ~azure.eventhub.EventHubClient + :rtype: ~azure.eventhub.client.EventHubClient """ log.info("{}: Starting {} clients".format(self.container_id, len(self.clients))) self._create_connection() - for client in self.clients: - client.open(connection=self.connection) - return self + try: + self._start_clients() + redirects = [c.redirected for c in self.clients if c.redirected] + failed = [c.error for c in self.clients if c.error] + if failed and len(failed) == len(self.clients): + log.warning("{}: All clients failed to start.".format(self.container_id)) + raise failed[0] + elif failed: + log.warning("{}: {} clients failed to start.".format(self.container_id, len(failed))) + elif redirects: + self._handle_redirect(redirects) + except EventHubError: + self.stop() + raise + except Exception as e: + self.stop() + raise EventHubError(str(e)) + return failed def stop(self): """ @@ -262,7 +304,7 @@ def get_eventhub_info(self): finally: mgmt_client.close() - def add_receiver(self, consumer_group, partition, offset=None, prefetch=300): + def add_receiver(self, consumer_group, partition, offset=None, prefetch=300, operation=None): """ Add a receiver to the client for a particular consumer group and partition. @@ -271,21 +313,22 @@ def add_receiver(self, consumer_group, partition, offset=None, prefetch=300): :param partition: The ID of the partition. :type partition: str :param offset: The offset from which to start receiving. - :type offset: ~azure.eventhub.Offset + :type offset: ~azure.eventhub.common.Offset :param prefetch: The message prefetch count of the receiver. Default is 300. :type prefetch: int - :rtype: ~azure.eventhub.Receiver + :rtype: ~azure.eventhub.receiver.Receiver """ + path = self.address.path + operation if operation else self.address.path source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( - self.address.hostname, self.address.path, consumer_group, partition) + self.address.hostname, path, consumer_group, partition) source = Source(source_url) if offset is not None: source.set_filter(offset.selector()) handler = Receiver(self, source, prefetch=prefetch) - self.clients.append(handler._handler) # pylint: disable=protected-access + self.clients.append(handler) return handler - def add_epoch_receiver(self, consumer_group, partition, epoch, prefetch=300): + def add_epoch_receiver(self, consumer_group, partition, epoch, prefetch=300, operation=None): """ Add a receiver to the client with an epoch value. Only a single epoch receiver can connect to a partition at any given time - additional epoch receivers must have @@ -300,26 +343,29 @@ def add_epoch_receiver(self, consumer_group, partition, epoch, prefetch=300): :type epoch: int :param prefetch: The message prefetch count of the receiver. Default is 300. :type prefetch: int - :rtype: ~azure.eventhub.Receiver + :rtype: ~azure.eventhub.receiver.Receiver """ + path = self.address.path + operation if operation else self.address.path source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( - self.address.hostname, self.address.path, consumer_group, partition) + self.address.hostname, path, consumer_group, partition) handler = Receiver(self, source_url, prefetch=prefetch, epoch=epoch) - self.clients.append(handler._handler) # pylint: disable=protected-access + self.clients.append(handler) return handler - def add_sender(self, partition=None): + def add_sender(self, partition=None, operation=None): """ - Add a sender to the client to send ~azure.eventhub.EventData object + Add a sender to the client to send ~azure.eventhub.common.EventData object to an EventHub. :param partition: Optionally specify a particular partition to send to. If omitted, the events will be distributed to available partitions via round-robin. :type parition: str - :rtype: ~azure.eventhub.Sender + :rtype: ~azure.eventhub.sender.Sender """ target = "amqps://{}{}".format(self.address.hostname, self.address.path) + if operation: + target = target + operation handler = Sender(self, target, partition=partition) - self.clients.append(handler._handler) # pylint: disable=protected-access + self.clients.append(handler) return handler diff --git a/azure/eventprocessorhost/eph.py b/azure/eventprocessorhost/eph.py index c344628..27cbc3e 100644 --- a/azure/eventprocessorhost/eph.py +++ b/azure/eventprocessorhost/eph.py @@ -25,7 +25,8 @@ def __init__(self, event_processor, eh_config, storage_manager, ep_params=None, :type eh_config: ~azure.eventprocessorhost.eh_config.EventHubConfig :param storage_manager: The Azure storage manager for persisting lease and checkpoint information. - :type storage_manager: ~azure.eventprocessorhost.azure_storage_checkpoint_manager.AzureStorageCheckpointLeaseManager + :type storage_manager: + ~azure.eventprocessorhost.azure_storage_checkpoint_manager.AzureStorageCheckpointLeaseManager :param ep_params: Optional arbitrary parameters to be passed into the event_processor on initialization. :type ep_params: list diff --git a/pylintrc b/pylintrc index 14cb341..7b3f956 100644 --- a/pylintrc +++ b/pylintrc @@ -6,7 +6,7 @@ reports=no # For all codes, run 'pylint --list-msgs' or go to 'https://pylint.readthedocs.io/en/latest/reference_guide/features.html' # locally-disabled: Warning locally suppressed using disable-msg # cyclic-import: because of https://github.com/PyCQA/pylint/issues/850 -disable=missing-docstring,locally-disabled,fixme,cyclic-import,too-many-arguments,invalid-name,duplicate-code,logging-format-interpolation,too-many-instance-attributes,too-few-public-methods +disable=raising-bad-type,missing-docstring,locally-disabled,fixme,cyclic-import,too-many-arguments,invalid-name,duplicate-code,logging-format-interpolation,too-many-instance-attributes,too-few-public-methods [FORMAT] max-line-length=120 diff --git a/setup.py b/setup.py index b6275ea..9fce5a2 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ zip_safe=False, packages=find_packages(exclude=["examples", "tests"]), install_requires=[ - 'uamqp==0.1.0rc2', + 'uamqp~=0.1.0', 'msrestazure~=0.4.11', 'azure-common~=1.1', 'azure-storage~=0.36.0' diff --git a/tests/__init__.py b/tests/__init__.py index 7ec7d3b..7b7c91a 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -12,7 +12,7 @@ def get_logger(filename, level=logging.INFO): azure_logger = logging.getLogger("azure") azure_logger.setLevel(level) uamqp_logger = logging.getLogger("uamqp") - uamqp_logger.setLevel(logging.DEBUG) + uamqp_logger.setLevel(logging.INFO) formatter = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s') console_handler = logging.StreamHandler(stream=sys.stdout) diff --git a/tests/test_longrunning_receive.py b/tests/test_longrunning_receive.py index b5519e3..60007a5 100644 --- a/tests/test_longrunning_receive.py +++ b/tests/test_longrunning_receive.py @@ -17,7 +17,7 @@ from urllib.parse import quote_plus from azure.eventhub import Offset -from azure.eventhub.async import EventHubClientAsync +from azure.eventhub import EventHubClientAsync try: import tests @@ -54,6 +54,7 @@ async def pump(_pid, receiver, _args, _dl): total)) except Exception as e: print("Partition {} receiver failed: {}".format(_pid, e)) + raise def test_long_running_receive(): @@ -73,7 +74,7 @@ def test_long_running_receive(): if args.conn_str: client = EventHubClientAsync.from_connection_string( args.conn_str, - eventhub=args.eventhub) + eventhub=args.eventhub, debug=False) elif args.address: client = EventHubClientAsync( args.address, @@ -97,8 +98,6 @@ def test_long_running_receive(): pumps.append(pump(pid, receiver, args, args.duration)) loop.run_until_complete(client.run_async()) loop.run_until_complete(asyncio.gather(*pumps)) - except: - raise finally: loop.run_until_complete(client.stop_async()) From 4d7b28d890dfb6b73594577d905d7a99a0186f4b Mon Sep 17 00:00:00 2001 From: annatisch Date: Thu, 5 Jul 2018 15:38:46 -0700 Subject: [PATCH 11/52] Fix long running test --- tests/test_negative.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_negative.py b/tests/test_negative.py index 3c8dde6..753f2e2 100644 --- a/tests/test_negative.py +++ b/tests/test_negative.py @@ -273,8 +273,10 @@ async def test_send_null_body_async(connection_str): async def pump(receiver): messages = 0 + count = 0 batch = await receiver.receive(timeout=10) - while batch: + while batch and count <= 5: + count += 1 messages += len(batch) batch = await receiver.receive(timeout=10) return messages From e174bd758dcf96e326a954ed4a9cafceafaaf100 Mon Sep 17 00:00:00 2001 From: annatisch Date: Fri, 6 Jul 2018 09:11:36 -0700 Subject: [PATCH 12/52] Docstring and sample cleanups --- HISTORY.rst | 3 +- azure/eventhub/_async/__init__.py | 13 +++++++-- azure/eventhub/_async/receiver_async.py | 26 ++++++++++++++++- azure/eventhub/_async/sender_async.py | 39 +++++++++++++++++++++++-- azure/eventhub/client.py | 16 +++++++++- azure/eventhub/common.py | 27 ++++++++--------- azure/eventhub/receiver.py | 34 +++++++++++++++++++-- azure/eventhub/sender.py | 34 +++++++++++++++++++-- examples/batch_send.py | 3 +- examples/batch_transfer.py | 3 +- examples/eph.py | 2 +- examples/recv.py | 3 +- examples/recv_async.py | 10 +++---- examples/recv_batch.py | 6 +--- examples/send.py | 1 + examples/send_async.py | 5 ++-- examples/transfer.py | 14 +++++++-- 17 files changed, 196 insertions(+), 43 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 9e63944..c90d34f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,12 +3,13 @@ Release History =============== -0.2.0rc1 (unreleased) +0.2.0rc1 (2018-07-06) +++++++++++++++++++++ - **Breaking change** Restructured library to support Python 3.7. Submodule `async` has been renamed and all classes from this module can now be imported from azure.eventhub directly. - **Breaking change** Removed optional `callback` argument from `Receiver.receive` and `AsyncReceiver.receive`. +- **Breaking change** `EventData.properties` has been renamed to `EventData.application_properties`. This removes the potential for messages to be processed via callback for not yet returned in the batch. - Updated uAMQP dependency to v0.1.0 diff --git a/azure/eventhub/_async/__init__.py b/azure/eventhub/_async/__init__.py index b17394c..e2e727a 100644 --- a/azure/eventhub/_async/__init__.py +++ b/azure/eventhub/_async/__init__.py @@ -117,8 +117,8 @@ async def run_async(self): Run the EventHubClient asynchronously. Opens the connection and starts running all AsyncSender/AsyncReceiver clients. Returns a list of the start up results. For a succcesful client start the - result will be `None`, otherwise the exception raise. - If all clients failed to start, the run will fail, shut down the connection + result will be `None`, otherwise the exception raised. + If all clients failed to start, then run will fail, shut down the connection and raise an exception. If at least one client starts up successfully the run command will succeed. @@ -193,6 +193,9 @@ def add_async_receiver(self, consumer_group, partition, offset=None, prefetch=30 :type offset: ~azure.eventhub.common.Offset :param prefetch: The message prefetch count of the receiver. Default is 300. :type prefetch: int + :operation: An optional operation to be appended to the hostname in the source URL. + The value must start with `/` character. + :type operation: str :rtype: ~azure.eventhub._async.receiver_async.ReceiverAsync """ path = self.address.path + operation if operation else self.address.path @@ -220,6 +223,9 @@ def add_async_epoch_receiver(self, consumer_group, partition, epoch, prefetch=30 :type epoch: int :param prefetch: The message prefetch count of the receiver. Default is 300. :type prefetch: int + :operation: An optional operation to be appended to the hostname in the source URL. + The value must start with `/` character. + :type operation: str :rtype: ~azure.eventhub._async.receiver_async.ReceiverAsync """ path = self.address.path + operation if operation else self.address.path @@ -238,6 +244,9 @@ def add_async_sender(self, partition=None, operation=None, loop=None): If omitted, the events will be distributed to available partitions via round-robin. :type partition: str + :operation: An optional operation to be appended to the hostname in the target URL. + The value must start with `/` character. + :type operation: str :rtype: ~azure.eventhub._async.sender_async.SenderAsync """ target = "amqps://{}{}".format(self.address.hostname, self.address.path) diff --git a/azure/eventhub/_async/receiver_async.py b/azure/eventhub/_async/receiver_async.py index 9438fae..2ceb518 100644 --- a/azure/eventhub/_async/receiver_async.py +++ b/azure/eventhub/_async/receiver_async.py @@ -52,6 +52,14 @@ def __init__(self, client, source, prefetch=300, epoch=None, loop=None): # pyli loop=self.loop) async def open_async(self, connection): + """ + Open the Receiver using the supplied conneciton. + If the handler has previously been redirected, the redirect + context will be used to create a new handler before opening it. + + :param connection: The underlying client shared connection. + :type: connection: ~uamqp._async.connection_async.ConnectionAsync + """ if self.redirected: self._handler = ReceiveClientAsync( self.redirected.address, @@ -64,6 +72,13 @@ async def open_async(self, connection): await self._handler.open_async(connection=connection) async def has_started(self): + """ + Whether the handler has completed all start up processes such as + establishing the connection, session, link and authentication, and + is not ready to process messages. + + :rtype: bool + """ # pylint: disable=protected-access timeout = False auth_in_progress = False @@ -79,6 +94,15 @@ async def has_started(self): return True async def close_async(self, exception=None): + """ + Close down the handler. If the handler has already closed, + this will be a no op. An optional exception can be passed in to + indicate that the handler was shutdown due to error. + + :param exception: An optional exception if the handler is closing + due to an error. + :type exception: Exception + """ if self.error: return elif isinstance(exception, errors.LinkRedirect): @@ -88,7 +112,7 @@ async def close_async(self, exception=None): elif exception: self.error = EventHubError(str(exception)) else: - self.error = EventHubError("This receive client is now closed.") + self.error = EventHubError("This receive handler is now closed.") await self._handler.close_async() async def receive(self, max_batch_size=None, timeout=None): diff --git a/azure/eventhub/_async/sender_async.py b/azure/eventhub/_async/sender_async.py index 27bce63..3e57e3c 100644 --- a/azure/eventhub/_async/sender_async.py +++ b/azure/eventhub/_async/sender_async.py @@ -18,7 +18,7 @@ class AsyncSender(Sender): def __init__(self, client, target, partition=None, loop=None): # pylint: disable=super-init-not-called """ - Instantiate an EventHub event SenderAsync client. + Instantiate an EventHub event SenderAsync handler. :param client: The parent EventHubClientAsync. :type client: ~azure.eventhub._async.EventHubClientAsync @@ -43,6 +43,14 @@ def __init__(self, client, target, partition=None, loop=None): # pylint: disabl self._condition = None async def open_async(self, connection): + """ + Open the Sender using the supplied conneciton. + If the handler has previously been redirected, the redirect + context will be used to create a new handler before opening it. + + :param connection: The underlying client shared connection. + :type: connection:~uamqp._async.connection_async.ConnectionAsync + """ if self.redirected: self._handler = SendClientAsync( self.redirected.address, @@ -52,6 +60,13 @@ async def open_async(self, connection): await self._handler.open_async(connection=connection) async def has_started(self): + """ + Whether the handler has completed all start up processes such as + establishing the connection, session, link and authentication, and + is not ready to process messages. + + :rtype: bool + """ # pylint: disable=protected-access timeout = False auth_in_progress = False @@ -67,6 +82,15 @@ async def has_started(self): return True async def close_async(self, exception=None): + """ + Close down the handler. If the handler has already closed, + this will be a no op. An optional exception can be passed in to + indicate that the handler was shutdown due to error. + + :param exception: An optional exception if the handler is closing + due to an error. + :type exception: Exception + """ if self.error: return elif isinstance(exception, errors.LinkRedirect): @@ -76,7 +100,7 @@ async def close_async(self, exception=None): elif exception: self.error = EventHubError(str(exception)) else: - self.error = EventHubError("This send client is now closed.") + self.error = EventHubError("This send handler is now closed.") await self._handler.close_async() async def send(self, event_data): @@ -108,3 +132,14 @@ async def send(self, event_data): raise error else: return self._outcome + + async def wait_async(self): + """ + Wait until all transferred events have been sent. + """ + if self.error: + raise self.error + try: + await self._handler.wait_async() + except Exception as e: + raise EventHubError("Send failed: {}".format(e)) diff --git a/azure/eventhub/client.py b/azure/eventhub/client.py index 91bc482..6e37dfe 100644 --- a/azure/eventhub/client.py +++ b/azure/eventhub/client.py @@ -232,8 +232,13 @@ def run(self): """ Run the EventHubClient in blocking mode. Opens the connection and starts running all Sender/Receiver clients. + Returns a list of the start up results. For a succcesful client start the + result will be `None`, otherwise the exception raised. + If all clients failed to start, then run will fail, shut down the connection + and raise an exception. + If at least one client starts up successfully the run command will succeed. - :rtype: ~azure.eventhub.client.EventHubClient + :rtype: list[~azure.eventhub.common.EventHubError] """ log.info("{}: Starting {} clients".format(self.container_id, len(self.clients))) self._create_connection() @@ -316,6 +321,9 @@ def add_receiver(self, consumer_group, partition, offset=None, prefetch=300, ope :type offset: ~azure.eventhub.common.Offset :param prefetch: The message prefetch count of the receiver. Default is 300. :type prefetch: int + :operation: An optional operation to be appended to the hostname in the source URL. + The value must start with `/` character. + :type operation: str :rtype: ~azure.eventhub.receiver.Receiver """ path = self.address.path + operation if operation else self.address.path @@ -343,6 +351,9 @@ def add_epoch_receiver(self, consumer_group, partition, epoch, prefetch=300, ope :type epoch: int :param prefetch: The message prefetch count of the receiver. Default is 300. :type prefetch: int + :operation: An optional operation to be appended to the hostname in the source URL. + The value must start with `/` character. + :type operation: str :rtype: ~azure.eventhub.receiver.Receiver """ path = self.address.path + operation if operation else self.address.path @@ -361,6 +372,9 @@ def add_sender(self, partition=None, operation=None): If omitted, the events will be distributed to available partitions via round-robin. :type parition: str + :operation: An optional operation to be appended to the hostname in the target URL. + The value must start with `/` character. + :type operation: str :rtype: ~azure.eventhub.sender.Sender """ target = "amqps://{}{}".format(self.address.hostname, self.address.path) diff --git a/azure/eventhub/common.py b/azure/eventhub/common.py index f528bba..4ba972a 100644 --- a/azure/eventhub/common.py +++ b/azure/eventhub/common.py @@ -36,25 +36,26 @@ def __init__(self, body=None, batch=None, to_device=None, message=None): """ self._partition_key = types.AMQPSymbol(EventData.PROP_PARTITION_KEY) self._annotations = {} - self._properties = {} - self._msg_properties = MessageProperties() + self._app_properties = {} + self.msg_properties = MessageProperties() if to_device: - self._msg_properties.to = '/devices/{}/messages/devicebound'.format(to_device) + self.msg_properties.to = '/devices/{}/messages/devicebound'.format(to_device) if batch: - self.message = BatchMessage(data=batch, multi_messages=True, properties=self._msg_properties) + self.message = BatchMessage(data=batch, multi_messages=True, properties=self.msg_properties) elif message: self.message = message + self.msg_properties = message.properties self._annotations = message.annotations - self._properties = message.application_properties + self._app_properties = message.application_properties else: if isinstance(body, list) and body: - self.message = Message(body[0], properties=self._msg_properties) + self.message = Message(body[0], properties=self.msg_properties) for more in body[1:]: self.message._body.append(more) # pylint: disable=protected-access elif body is None: raise ValueError("EventData cannot be None.") else: - self.message = Message(body, properties=self._msg_properties) + self.message = Message(body, properties=self.msg_properties) @property @@ -129,24 +130,24 @@ def partition_key(self, value): self._annotations = annotations @property - def properties(self): + def application_properties(self): """ Application defined properties on the message. :rtype: dict """ - return self._properties + return self._app_properties - @properties.setter - def properties(self, value): + @application_properties.setter + def application_properties(self, value): """ Application defined properties on the message. :param value: The application properties for the EventData. :type value: dict """ - self._properties = value - properties = dict(self._properties) + self._app_properties = value + properties = dict(self._app_properties) self.message.application_properties = properties @property diff --git a/azure/eventhub/receiver.py b/azure/eventhub/receiver.py index beec103..3cef829 100644 --- a/azure/eventhub/receiver.py +++ b/azure/eventhub/receiver.py @@ -48,6 +48,14 @@ def __init__(self, client, source, prefetch=300, epoch=None): timeout=self.timeout) def open(self, connection): + """ + Open the Receiver using the supplied conneciton. + If the handler has previously been redirected, the redirect + context will be used to create a new handler before opening it. + + :param connection: The underlying client shared connection. + :type: connection: ~uamqp.connection.Connection + """ if self.redirected: self._handler = ReceiveClient( self.redirected.address, @@ -59,10 +67,23 @@ def open(self, connection): self._handler.open(connection) def get_handler_state(self): + """ + Get the state of the underlying handler with regards to start + up processes. + + :rtype: ~uamqp.constants.MessageReceiverState + """ # pylint: disable=protected-access return self._handler._message_receiver.get_state() def has_started(self): + """ + Whether the handler has completed all start up processes such as + establishing the connection, session, link and authentication, and + is not ready to process messages. + + :rtype: bool + """ # pylint: disable=protected-access timeout = False auth_in_progress = False @@ -78,6 +99,15 @@ def has_started(self): return True def close(self, exception=None): + """ + Close down the handler. If the handler has already closed, + this will be a no op. An optional exception can be passed in to + indicate that the handler was shutdown due to error. + + :param exception: An optional exception if the handler is closing + due to an error. + :type exception: Exception + """ if self.error: return elif isinstance(exception, errors.LinkRedirect): @@ -87,13 +117,13 @@ def close(self, exception=None): elif exception: self.error = EventHubError(str(exception)) else: - self.error = EventHubError("This receive client is now closed.") + self.error = EventHubError("This receive handler is now closed.") self._handler.close() @property def queue_size(self): """ - The current size of the unprocessed message queue. + The current size of the unprocessed Event queue. :rtype: int """ diff --git a/azure/eventhub/sender.py b/azure/eventhub/sender.py index 0a0e08c..358a336 100644 --- a/azure/eventhub/sender.py +++ b/azure/eventhub/sender.py @@ -17,7 +17,7 @@ class Sender: def __init__(self, client, target, partition=None): """ - Instantiate an EventHub event Sender client. + Instantiate an EventHub event Sender handler. :param client: The parent EventHubClient. :type client: ~azure.eventhub.client.EventHubClient. @@ -39,6 +39,14 @@ def __init__(self, client, target, partition=None): self._condition = None def open(self, connection): + """ + Open the Sender using the supplied conneciton. + If the handler has previously been redirected, the redirect + context will be used to create a new handler before opening it. + + :param connection: The underlying client shared connection. + :type: connection: ~uamqp.connection.Connection + """ if self.redirected: self._handler = SendClient( self.redirected.address, @@ -48,10 +56,23 @@ def open(self, connection): self._handler.open(connection) def get_handler_state(self): + """ + Get the state of the underlying handler with regards to start + up processes. + + :rtype: ~uamqp.constants.MessageSenderState + """ # pylint: disable=protected-access return self._handler._message_sender.get_state() def has_started(self): + """ + Whether the handler has completed all start up processes such as + establishing the connection, session, link and authentication, and + is not ready to process messages. + + :rtype: bool + """ # pylint: disable=protected-access timeout = False auth_in_progress = False @@ -67,6 +88,15 @@ def has_started(self): return True def close(self, exception=None): + """ + Close down the handler. If the handler has already closed, + this will be a no op. An optional exception can be passed in to + indicate that the handler was shutdown due to error. + + :param exception: An optional exception if the handler is closing + due to an error. + :type exception: Exception + """ if self.error: return elif isinstance(exception, errors.LinkRedirect): @@ -76,7 +106,7 @@ def close(self, exception=None): elif exception: self.error = EventHubError(str(exception)) else: - self.error = EventHubError("This send client is now closed.") + self.error = EventHubError("This send handler is now closed.") self._handler.close() def send(self, event_data): diff --git a/examples/batch_send.py b/examples/batch_send.py index 3f48669..7cbf625 100644 --- a/examples/batch_send.py +++ b/examples/batch_send.py @@ -28,7 +28,8 @@ def data_generator(): - for i in range(15000): + for i in range(1500): + logger.info("Yielding message {}".format(i)) yield b"Hello world" diff --git a/examples/batch_transfer.py b/examples/batch_transfer.py index 99efd9e..676ac6c 100644 --- a/examples/batch_transfer.py +++ b/examples/batch_transfer.py @@ -33,7 +33,8 @@ def callback(outcome, condition): def data_generator(): - for i in range(15000): + for i in range(1500): + logger.info("Yielding message {}".format(i)) yield b"Hello world" diff --git a/examples/eph.py b/examples/eph.py index 7a9633c..39f0fbb 100644 --- a/examples/eph.py +++ b/examples/eph.py @@ -79,7 +79,7 @@ async def wait_and_close(host): """ Run EventProcessorHost for 2 minutes then shutdown. """ - await asyncio.sleep(120) + await asyncio.sleep(60) await host.close_async() diff --git a/examples/recv.py b/examples/recv.py index 25e2d34..92a5df2 100644 --- a/examples/recv.py +++ b/examples/recv.py @@ -38,9 +38,10 @@ receiver = client.add_receiver(CONSUMER_GROUP, PARTITION, prefetch=5000, offset=OFFSET) client.run() start_time = time.time() - for event_data in receiver.receive(timeout=10): + for event_data in receiver.receive(timeout=100): last_offset = event_data.offset last_sn = event_data.sequence_number + print("Received: {}, {}".format(last_offset, last_sn)) total += 1 end_time = time.time() diff --git a/examples/recv_async.py b/examples/recv_async.py index d025bc9..ab8da39 100644 --- a/examples/recv_async.py +++ b/examples/recv_async.py @@ -29,17 +29,17 @@ KEY = os.environ.get('EVENT_HUB_SAS_KEY') CONSUMER_GROUP = "$default" OFFSET = Offset("-1") -PARTITION = "0" -async def pump(client): - receiver = client.add_async_receiver(CONSUMER_GROUP, PARTITION, OFFSET, prefetch=5) +async def pump(client, partition): + receiver = client.add_async_receiver(CONSUMER_GROUP, partition, OFFSET, prefetch=5) await client.run_async() total = 0 start_time = time.time() for event_data in await receiver.receive(timeout=10): last_offset = event_data.offset last_sn = event_data.sequence_number + print("Received: {}, {}".format(last_offset, last_sn)) total += 1 end_time = time.time() run_time = end_time - start_time @@ -52,8 +52,8 @@ async def pump(client): loop = asyncio.get_event_loop() client = EventHubClientAsync(ADDRESS, debug=False, username=USER, password=KEY) tasks = [ - asyncio.ensure_future(pump(client)), - asyncio.ensure_future(pump(client))] + asyncio.ensure_future(pump(client, "0")), + asyncio.ensure_future(pump(client, "1"))] loop.run_until_complete(asyncio.wait(tasks)) loop.run_until_complete(client.stop_async()) loop.close() diff --git a/examples/recv_batch.py b/examples/recv_batch.py index f6373e4..7ce562d 100644 --- a/examples/recv_batch.py +++ b/examples/recv_batch.py @@ -31,10 +31,6 @@ PARTITION = "0" -def on_event_data(event_data): - logger.debug("Got event no. {}".format(event_data.sequence_number)) - - total = 0 last_sn = -1 last_offset = "-1" @@ -42,7 +38,7 @@ def on_event_data(event_data): try: receiver = client.add_receiver(CONSUMER_GROUP, PARTITION, prefetch=100, offset=OFFSET) client.run() - batched_events = receiver.receive(max_batch_size=10, callback=on_event_data) + batched_events = receiver.receive(max_batch_size=10) for event_data in batched_events: last_offset = event_data.offset last_sn = event_data.sequence_number diff --git a/examples/send.py b/examples/send.py index 9ba01f6..fe1d91c 100644 --- a/examples/send.py +++ b/examples/send.py @@ -37,6 +37,7 @@ try: start_time = time.time() for i in range(100): + logger.info("Sending message: {}".format(i)) sender.send(EventData(str(i))) except: raise diff --git a/examples/send_async.py b/examples/send_async.py index 96a6ce1..248fdcf 100644 --- a/examples/send_async.py +++ b/examples/send_async.py @@ -35,9 +35,10 @@ async def run(client): async def send(snd, count): for i in range(count): - data = EventData(str(i) + logger.info("Sending message: {}".format(i)) + data = EventData(str(i)) data.partition_key = b'SamplePartitionKey' - await snd.send(data)) + await snd.send(data) try: if not ADDRESS: diff --git a/examples/transfer.py b/examples/transfer.py index 5a37dda..5190add 100644 --- a/examples/transfer.py +++ b/examples/transfer.py @@ -27,18 +27,26 @@ USER = os.environ.get('EVENT_HUB_SAS_POLICY') KEY = os.environ.get('EVENT_HUB_SAS_KEY') + +def callback(outcome, condition): + logger.info("Message sent. Outcome: {}, Condition: {}".format( + outcome, condition)) + + try: if not ADDRESS: raise ValueError("No EventHubs URL supplied.") client = EventHubClient(ADDRESS, debug=False, username=USER, password=KEY) - sender = client.add_sender() + sender = client.add_sender(partition="1") client.run() try: start_time = time.time() - for i in range(1000): - sender.transfer(EventData(str(i))) + for i in range(100): + sender.transfer(EventData(str(i)), callback=callback) + logger.info("Queued 100 messages.") sender.wait() + logger.info("Finished processing queue.") except: raise finally: From 121ddef46668ce16342097192ef090d018bdce7c Mon Sep 17 00:00:00 2001 From: annatisch Date: Fri, 13 Jul 2018 16:12:04 -0700 Subject: [PATCH 13/52] Working on error retry --- azure/eventhub/common.py | 40 +++++++++++++++++++++-- azure/eventhub/sender.py | 27 ++++++++++++++-- features/eventhub.feature | 13 ++++---- features/steps/eventhub.py | 18 ++++++++++- features/steps/test_utils.py | 63 ++++++++++++++++++++++++++++++++++++ 5 files changed, 149 insertions(+), 12 deletions(-) diff --git a/azure/eventhub/common.py b/azure/eventhub/common.py index 4ba972a..7098ef6 100644 --- a/azure/eventhub/common.py +++ b/azure/eventhub/common.py @@ -7,7 +7,7 @@ import time from uamqp import Message, BatchMessage -from uamqp import types +from uamqp import types, constants from uamqp.message import MessageHeader, MessageProperties @@ -208,4 +208,40 @@ class EventHubError(Exception): """ Represents an error happened in the client. """ - pass + + def __init__(self, message, details=None): + self.error = None + self.message = message + self.details = [] + if isinstance(message, constants.MessageSendResult): + self.message = "Message send failed with result: {}".format(message) + if details and isinstance(details, list) and isinstance(details[0], list): + self.details = details[0] + self.error = details[0][0] + try: + self._parse_error(details[0]) + except: + raise + if self.error: + self.message += "\nError: {}".format(self.error) + for detail in self.details: + self.message += "\n{}".format(detail) + super(EventHubError, self).__init__(self.message) + + def _parse_error(self, error_list): + details = [] + _, _, self.error = error_list[0].decode('UTF-8').partition(':') + self.message = error_list[1].decode('UTF-8') + details_index = self.message.find(" Reference:") + if details_index >= 0: + details_msg = self.message[details_index + 1:] + self.message = self.message[0:details_index] + + tracking_index = details_msg.index(", TrackingId:") + system_index = details_msg.index(", SystemTracker:") + timestamp_index = details_msg.index(", Timestamp:") + details.append(details_msg[:tracking_index]) + details.append(details_msg[tracking_index + 2: system_index]) + details.append(details_msg[system_index + 2: timestamp_index]) + details.append(details_msg[timestamp_index + 2:]) + self.details = details \ No newline at end of file diff --git a/azure/eventhub/sender.py b/azure/eventhub/sender.py index 358a336..0aa3e77 100644 --- a/azure/eventhub/sender.py +++ b/azure/eventhub/sender.py @@ -3,6 +3,9 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- +import time + +import uamqp from uamqp import constants, errors from uamqp import SendClient @@ -28,13 +31,15 @@ def __init__(self, client, target, partition=None): self.error = None self.debug = client.debug self.partition = partition + self.retry_policy = uamqp.sender.RetryPolicy(max_retries=3, on_error=self._error_handler) if partition: target += "/Partitions/" + partition self._handler = SendClient( target, auth=client.auth, debug=self.debug, - msg_timeout=Sender.TIMEOUT) + msg_timeout=Sender.TIMEOUT, + retry_policy=self.retry_policy) self._outcome = None self._condition = None @@ -52,7 +57,8 @@ def open(self, connection): self.redirected.address, auth=None, debug=self.debug, - msg_timeout=Sender.TIMEOUT) + msg_timeout=Sender.TIMEOUT, + retry_policy=self.retry_policy) self._handler.open(connection) def get_handler_state(self): @@ -180,6 +186,23 @@ def _on_outcome(self, outcome, condition): self._outcome = outcome self._condition = condition + def _error_handler(self, error): + """ + Called internally when an event has failed to send so we + can parse the error to determine whether we should attempt + to retry sending the event again. + Returns the action to take according to error type. + + :param error: The error received in the send attempt. + :type error: list[list[bytes]] + :rtype: ~uamqp.sender.SendFailedAction + """ + if isinstance(error, list) and isinstance(error[0], list): + error_type = error[0][0].decode('UTF-8') + if error_type == 'com.microsoft:server-busy': + return uamqp.sender.SendFailedAction(retry=True, backoff=4) + return uamqp.sender.SendFailedAction(retry=True, backoff=4) + @staticmethod def _error(outcome, condition): return None if outcome == constants.MessageSendResult.Ok else EventHubError(outcome, condition) diff --git a/features/eventhub.feature b/features/eventhub.feature index c96a998..19bf68b 100644 --- a/features/eventhub.feature +++ b/features/eventhub.feature @@ -5,15 +5,14 @@ Feature: Exercising EventHub SDK -# Scenario: Just sends for 3 days, no receives. Focus on send failures only. - @long-running - Scenario: Generic send and receive on client for 3 days. - Given the EventHub SDK is installed - And an EventHub is created with credentials retrieved - When I send and receive messages for 72 hours + Scenario: Just sends for 3 days, no receives. Focus on send failures only. + Given The EventHub SDK is installed + And An EventHub is created with credentials retrieved + When I start a message sender + And I send messages for 72 hours Then I should receive no errors - And I can shutdown the sender and receiver cleanly + And I can shutdown sender And I remove the EventHub # Scenario: Sender stays idle for 45 minutes and sends some number of messages after each idle duration. diff --git a/features/steps/eventhub.py b/features/steps/eventhub.py index b92bdc5..01cda1d 100644 --- a/features/steps/eventhub.py +++ b/features/steps/eventhub.py @@ -5,6 +5,7 @@ import asyncio import uuid +import functools from behave import * @@ -24,7 +25,22 @@ def step_impl(context): def step_impl(context, properties): #from mgmt_settings_real import get_credentials, SUBSCRIPTION_ID #rg, mgmt_client = test_utils.create_mgmt_client(get_credentials(), SUBSCRIPTION_ID) - context.eh_config = test_utils.get_eventhub_config() + _, prop = properties.split(' ') + if prop == '100TU': + context.eh_config = test_utils.get_eventhub_100TU_config() + else: + raise ValueError("Unrecognised property: {}".format(prop)) + +@When('I start a message sender') +def step_impl(context): + from azure.eventhub import EventHubClient + address = "sb://{}/{}".format(context.eh_config['hostname'], context.eh_config['event_hub']) + context.client = EventHubClient( + address, + username=context.eh_config['key_name'], + password=context.eh_config['access_key']) + context.sender = client.add_sender() + context.client.run() @when('I {clients} messages for {hours} hours') def step_impl(context, clients, hours): diff --git a/features/steps/test_utils.py b/features/steps/test_utils.py index 31eb7b5..7cc4d34 100644 --- a/features/steps/test_utils.py +++ b/features/steps/test_utils.py @@ -4,6 +4,8 @@ # -------------------------------------------------------------------------------------------- import uuid +import time +import asyncio def create_mgmt_client(credentials, subscription, location='westus'): from azure.mgmt.resource import ResourceManagementClient @@ -32,3 +34,64 @@ def get_eventhub_config(): config['consumer_group'] = "$Default" config['partition'] = "0" return config + + +def get_eventhub_100TU_config(): + config = {} + config['hostname'] = os.environ['EVENT_HUB_100TU_HOSTNAME'] + config['event_hub'] = os.environ['EVENT_HUB_100TU_NAME'] + config['key_name'] = os.environ['EVENT_HUB_100TU_SAS_POLICY'] + config['access_key'] = os.environ['EVENT_HUB_100TU_SAS_KEY'] + config['consumer_group'] = "$Default" + config['partition'] = "0" + return config + + +def send_constant_messages(sender, timeout, payload=1024): + deadline = time.time() + total = 0 + while time.time() < deadline: + data = EventData(body=b"D" * payload) + sender.send(data) + total += 1 + return total + + +def send_constant_async_messages(sender, timeout, batch_size=10000, payload=1024): + deadline = time.time() + total = 0 + while time.time() < deadline: + data = EventData(body=b"D" * args.payload) + sender.transfer(data) + total += 1 + if total % 10000 == 0: + sender.wait() + return total + + +def send_constant_async_messages(sender, timeout, batch_size=1, payload=1024): + deadline = time.time() + while time.time() < deadline: + if batch_size > 1: + data = EventData(batch=data_generator()) + else: + data = EventData(body=b"D" * payload) + + +async def receive_pump(receiver, timeout, validation=True): + total = 0 + deadline = time.time() + timeout + sequence = 0 + offset = None + while time.time() < deadline: + batch = await receiver.receive(timeout=5) + total += len(batch) + if validation: + assert receiver.offset + for event in batch: + next_sequence = event.sequence_number + assert next_sequence > sequence, "Received Event with lower sequence number than previous." + assert (next_sequence - sequence) == 1, "Sequence number skipped by a value great than 1." + sequence = next_sequence + msg_data = b"".join([b for b in event.body]).decode('UTF-8') + assert json.loads(msg_data), "Unable to deserialize Event data." From f480b0d430228702a597eaffe624082161070988 Mon Sep 17 00:00:00 2001 From: annatisch Date: Thu, 19 Jul 2018 13:10:59 -0700 Subject: [PATCH 14/52] Improved error processing --- azure/eventhub/_async/__init__.py | 14 ++++++- azure/eventhub/_async/receiver_async.py | 10 ++++- azure/eventhub/_async/sender_async.py | 18 ++++++++- azure/eventhub/client.py | 4 +- azure/eventhub/common.py | 49 ++++++++++++++++++------- azure/eventhub/receiver.py | 2 +- azure/eventhub/sender.py | 41 ++++++++++----------- 7 files changed, 95 insertions(+), 43 deletions(-) diff --git a/azure/eventhub/_async/__init__.py b/azure/eventhub/_async/__init__.py index e2e727a..734e7b4 100644 --- a/azure/eventhub/_async/__init__.py +++ b/azure/eventhub/_async/__init__.py @@ -51,7 +51,7 @@ def _create_auth(self, auth_uri, username, password): # pylint: disable=no-self """ if "@sas.root" in username: return authentication.SASLPlain(self.address.hostname, username, password) - return authentication.SASTokenAsync.from_shared_access_key(auth_uri, username, password) + return authentication.SASTokenAsync.from_shared_access_key(auth_uri, username, password, timeout=60) def _create_connection_async(self): """ @@ -109,6 +109,10 @@ async def _handle_redirect(self, redirects): if not all(r.hostname == redirects[0].hostname for r in redirects): raise EventHubError("Multiple clients attempting to redirect to different hosts.") self.auth = self._create_auth(redirects[0].address.decode('utf-8'), **self._auth_config) + #port = str(redirects[0].port).encode('UTF-8') + #path = self.address.path.encode('UTF-8') + #self.mgmt_node = b"pyot/$management" #+ redirects[0].hostname b"amqps://pyot.azure-devices.net" + b":" + port + + #print("setting mgmt node", self.mgmt_node) await self.connection.redirect_async(redirects[0], self.auth) await asyncio.gather(*[c.open_async(self.connection) for c in self.clients]) @@ -161,14 +165,18 @@ async def get_eventhub_info_async(self): :rtype: dict """ + self._create_connection_async() eh_name = self.address.path.lstrip('/') target = "amqps://{}/{}".format(self.address.hostname, eh_name) - async with AMQPClientAsync(target, auth=self.auth, debug=self.debug) as mgmt_client: + try: + mgmt_client = AMQPClientAsync(target, auth=self.auth, debug=self.debug) + await mgmt_client.open_async(connection=self.connection) mgmt_msg = Message(application_properties={'name': eh_name}) response = await mgmt_client.mgmt_request_async( mgmt_msg, constants.READ_OPERATION, op_type=b'com.microsoft:eventhub', + node=self.mgmt_node, status_code_field=b'status-code', description_fields=b'status-description') eh_info = response.get_data() @@ -180,6 +188,8 @@ async def get_eventhub_info_async(self): output['partition_count'] = eh_info[b'partition_count'] output['partition_ids'] = [p.decode('utf-8') for p in eh_info[b'partition_ids']] return output + finally: + await mgmt_client.close_async() def add_async_receiver(self, consumer_group, partition, offset=None, prefetch=300, operation=None, loop=None): """ diff --git a/azure/eventhub/_async/receiver_async.py b/azure/eventhub/_async/receiver_async.py index 2ceb518..b3b5138 100644 --- a/azure/eventhub/_async/receiver_async.py +++ b/azure/eventhub/_async/receiver_async.py @@ -88,7 +88,7 @@ async def has_started(self): raise EventHubError("Authorization timeout.") elif auth_in_progress: return False - elif not await self._handler._client_ready(): + elif not await self._handler._client_ready_async(): return False else: return True @@ -109,6 +109,8 @@ async def close_async(self, exception=None): self.redirected = exception elif isinstance(exception, EventHubError): self.error = exception + elif isinstance(exception, (errors.LinkDetach, errors.ConnectionClose)): + self.error = EventHubError(str(exception), exception) elif exception: self.error = EventHubError(str(exception)) else: @@ -141,7 +143,11 @@ async def receive(self, max_batch_size=None, timeout=None): data_batch.append(event_data) return data_batch except errors.LinkDetach as detach: - error = EventHubError(str(detach)) + error = EventHubError(str(detach), detach) + await self.close_async(exception=error) + raise error + except errors.ConnectionClose as close: + error = EventHubError(str(close), close) await self.close_async(exception=error) raise error except Exception as e: diff --git a/azure/eventhub/_async/sender_async.py b/azure/eventhub/_async/sender_async.py index 3e57e3c..abcd0fa 100644 --- a/azure/eventhub/_async/sender_async.py +++ b/azure/eventhub/_async/sender_async.py @@ -76,7 +76,7 @@ async def has_started(self): raise EventHubError("Authorization timeout.") elif auth_in_progress: return False - elif not await self._handler._client_ready(): + elif not await self._handler._client_ready_async(): return False else: return True @@ -97,6 +97,8 @@ async def close_async(self, exception=None): self.redirected = exception elif isinstance(exception, EventHubError): self.error = exception + elif isinstance(exception, (errors.LinkDetach, errors.ConnectionClose)): + self.error = EventHubError(str(error), error) elif exception: self.error = EventHubError(str(exception)) else: @@ -123,7 +125,11 @@ async def send(self, event_data): if self._outcome != constants.MessageSendResult.Ok: raise Sender._error(self._outcome, self._condition) except errors.LinkDetach as detach: - error = EventHubError(str(detach)) + error = EventHubError(str(detach), detach) + await self.close_async(exception=error) + raise error + except errors.ConnectionClose as close: + error = EventHubError(str(close), close) await self.close_async(exception=error) raise error except Exception as e: @@ -141,5 +147,13 @@ async def wait_async(self): raise self.error try: await self._handler.wait_async() + except errors.LinkDetach as detach: + error = EventHubError(str(detach), detach) + await self.close_async(exception=error) + raise error + except errors.ConnectionClose as close: + error = EventHubError(str(close), close) + await self.close_async(exception=error) + raise error except Exception as e: raise EventHubError("Send failed: {}".format(e)) diff --git a/azure/eventhub/client.py b/azure/eventhub/client.py index 6e37dfe..8a47a03 100644 --- a/azure/eventhub/client.py +++ b/azure/eventhub/client.py @@ -108,6 +108,7 @@ def __init__(self, address, username=None, password=None, debug=False): """ self.container_id = "eventhub.pysdk-" + str(uuid.uuid4())[:8] self.address = urlparse(address) + self.mgmt_node = b"$management" url_username = unquote_plus(self.address.username) if self.address.username else None username = username or url_username url_password = unquote_plus(self.address.password) if self.address.password else None @@ -147,6 +148,7 @@ def from_iothub_connection_string(cls, conn_str, **kwargs): password = _generate_sas_token(address, policy, key) client = cls("amqps://" + address, username=username, password=password, **kwargs) client._auth_config = {'username': policy, 'password': key} # pylint: disable=protected-access + client.mgmt_node = ("amqps://" + address + ":5671/pyot/$management").encode('UTF-8') return client def _create_auth(self, auth_uri, username, password): # pylint: disable=no-self-use @@ -163,7 +165,7 @@ def _create_auth(self, auth_uri, username, password): # pylint: disable=no-self """ if "@sas.root" in username: return authentication.SASLPlain(self.address.hostname, username, password) - return authentication.SASTokenAuth.from_shared_access_key(auth_uri, username, password) + return authentication.SASTokenAuth.from_shared_access_key(auth_uri, username, password, timeout=60) def _create_properties(self): # pylint: disable=no-self-use """ diff --git a/azure/eventhub/common.py b/azure/eventhub/common.py index 7098ef6..9c3c2f4 100644 --- a/azure/eventhub/common.py +++ b/azure/eventhub/common.py @@ -7,10 +7,30 @@ import time from uamqp import Message, BatchMessage -from uamqp import types, constants +from uamqp import types, constants, errors from uamqp.message import MessageHeader, MessageProperties +def _error_handler(error): + """ + Called internally when an event has failed to send so we + can parse the error to determine whether we should attempt + to retry sending the event again. + Returns the action to take according to error type. + + :param error: The error received in the send attempt. + :type error: Exception + :rtype: ~uamqp.errors.ErrorAction + """ + if error.condition == b'com.microsoft:server-busy': + return errors.ErrorAction(retry=True, backoff=4) + elif error.condition == b'com.microsoft:timeout': + return errors.ErrorAction(retry=True, backoff=2) + elif error.condition == b'com.microsoft:operation-cancelled': + return errors.ErrorAction(retry=True) + return errors.ErrorAction(retry=True) + + class EventData(object): """ The EventData class is a holder of event content. @@ -212,26 +232,27 @@ class EventHubError(Exception): def __init__(self, message, details=None): self.error = None self.message = message - self.details = [] + self.details = details if isinstance(message, constants.MessageSendResult): self.message = "Message send failed with result: {}".format(message) - if details and isinstance(details, list) and isinstance(details[0], list): - self.details = details[0] - self.error = details[0][0] + if details and isinstance(details, Exception): try: - self._parse_error(details[0]) - except: - raise - if self.error: + condition = details.condition.value.decode('UTF-8') + except AttributeError: + condition = details.condition.decode('UTF-8') + _, _, self.error = condition.partition(':') self.message += "\nError: {}".format(self.error) - for detail in self.details: - self.message += "\n{}".format(detail) + try: + self._parse_error(details.description) + for detail in self.details: + self.message += "\n{}".format(detail) + except: + self.message += "\n{}".format(details) super(EventHubError, self).__init__(self.message) def _parse_error(self, error_list): details = [] - _, _, self.error = error_list[0].decode('UTF-8').partition(':') - self.message = error_list[1].decode('UTF-8') + self.message = error_list if isinstance(error_list, str) else error_list.decode('UTF-8') details_index = self.message.find(" Reference:") if details_index >= 0: details_msg = self.message[details_index + 1:] @@ -244,4 +265,4 @@ def _parse_error(self, error_list): details.append(details_msg[tracking_index + 2: system_index]) details.append(details_msg[system_index + 2: timestamp_index]) details.append(details_msg[timestamp_index + 2:]) - self.details = details \ No newline at end of file + self.details = details diff --git a/azure/eventhub/receiver.py b/azure/eventhub/receiver.py index 3cef829..a21b30f 100644 --- a/azure/eventhub/receiver.py +++ b/azure/eventhub/receiver.py @@ -158,7 +158,7 @@ def receive(self, max_batch_size=None, timeout=None): data_batch.append(event_data) return data_batch except errors.LinkDetach as detach: - error = EventHubError(str(detach)) + error = EventHubError(str(detach), detach) self.close(exception=error) raise error except Exception as e: diff --git a/azure/eventhub/sender.py b/azure/eventhub/sender.py index 0aa3e77..83a6c4c 100644 --- a/azure/eventhub/sender.py +++ b/azure/eventhub/sender.py @@ -9,7 +9,7 @@ from uamqp import constants, errors from uamqp import SendClient -from azure.eventhub.common import EventHubError +from azure.eventhub.common import EventHubError, _error_handler class Sender: @@ -31,7 +31,7 @@ def __init__(self, client, target, partition=None): self.error = None self.debug = client.debug self.partition = partition - self.retry_policy = uamqp.sender.RetryPolicy(max_retries=3, on_error=self._error_handler) + self.retry_policy = errors.ErrorPolicy(max_retries=3, on_error=_error_handler) if partition: target += "/Partitions/" + partition self._handler = SendClient( @@ -39,7 +39,7 @@ def __init__(self, client, target, partition=None): auth=client.auth, debug=self.debug, msg_timeout=Sender.TIMEOUT, - retry_policy=self.retry_policy) + error_policy=self.retry_policy) self._outcome = None self._condition = None @@ -136,8 +136,16 @@ def send(self, event_data): self._handler.send_message(event_data.message) if self._outcome != constants.MessageSendResult.Ok: raise Sender._error(self._outcome, self._condition) + except errors.MessageException as failed: + error = EventHubError(str(failed), failed) + self.close(exception=error) + raise error except errors.LinkDetach as detach: - error = EventHubError(str(detach)) + error = EventHubError(str(detach), detach) + self.close(exception=error) + raise error + except errors.ConnectionClose as close: + error = EventHubError(str(close), close) self.close(exception=error) raise error except Exception as e: @@ -173,6 +181,14 @@ def wait(self): raise self.error try: self._handler.wait() + except errors.LinkDetach as detach: + error = EventHubError(str(detach), detach) + self.close(exception=error) + raise error + except errors.ConnectionClose as close: + error = EventHubError(str(close), close) + self.close(exception=error) + raise error except Exception as e: raise EventHubError("Send failed: {}".format(e)) @@ -186,23 +202,6 @@ def _on_outcome(self, outcome, condition): self._outcome = outcome self._condition = condition - def _error_handler(self, error): - """ - Called internally when an event has failed to send so we - can parse the error to determine whether we should attempt - to retry sending the event again. - Returns the action to take according to error type. - - :param error: The error received in the send attempt. - :type error: list[list[bytes]] - :rtype: ~uamqp.sender.SendFailedAction - """ - if isinstance(error, list) and isinstance(error[0], list): - error_type = error[0][0].decode('UTF-8') - if error_type == 'com.microsoft:server-busy': - return uamqp.sender.SendFailedAction(retry=True, backoff=4) - return uamqp.sender.SendFailedAction(retry=True, backoff=4) - @staticmethod def _error(outcome, condition): return None if outcome == constants.MessageSendResult.Ok else EventHubError(outcome, condition) From b681df5e6eb9278a1390bbf87757a3caeb0c2e64 Mon Sep 17 00:00:00 2001 From: annatisch Date: Thu, 19 Jul 2018 13:30:43 -0700 Subject: [PATCH 15/52] Fixed partition manager --- azure/eventprocessorhost/partition_manager.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/azure/eventprocessorhost/partition_manager.py b/azure/eventprocessorhost/partition_manager.py index 2ae402e..da58423 100644 --- a/azure/eventprocessorhost/partition_manager.py +++ b/azure/eventprocessorhost/partition_manager.py @@ -35,15 +35,17 @@ async def get_partition_ids_async(self): :rtype: list[str] """ if not self.partition_ids: - eh_client = EventHubClientAsync( - self.host.eh_config.client_address, - debug=self.host.eph_options.debug_trace) try: - eh_info = await eh_client.get_eventhub_info_async() - self.partition_ids = eh_info['partition_ids'] - except Exception as err: # pylint: disable=broad-except - raise Exception("Failed to get partition ids", repr(err)) - + eh_client = EventHubClientAsync( + self.host.eh_config.client_address, + debug=self.host.eph_options.debug_trace) + try: + eh_info = await eh_client.get_eventhub_info_async() + self.partition_ids = eh_info['partition_ids'] + except Exception as err: # pylint: disable=broad-except + raise Exception("Failed to get partition ids", repr(err)) + finally: + await eh_client.stop_async() return self.partition_ids async def start_async(self): From 90a9e774e30dc6cff45370fd8c5486b4af3980aa Mon Sep 17 00:00:00 2001 From: annatisch Date: Fri, 20 Jul 2018 07:32:33 -0700 Subject: [PATCH 16/52] Progress on IotHub error --- azure/eventhub/_async/__init__.py | 26 ++++++++++++--------- azure/eventhub/client.py | 31 ++++++++++++++++--------- azure/eventhub/common.py | 2 ++ azure/eventhub/sender.py | 36 +++++++++++++++++++++++++----- conftest.py | 2 +- tests/__init__.py | 2 +- tests/test_iothub_receive_async.py | 15 +++++-------- tests/test_iothub_send.py | 2 ++ tests/test_negative.py | 6 +++-- tests/test_receive_async.py | 2 ++ 10 files changed, 84 insertions(+), 40 deletions(-) diff --git a/azure/eventhub/_async/__init__.py b/azure/eventhub/_async/__init__.py index 734e7b4..e961302 100644 --- a/azure/eventhub/_async/__init__.py +++ b/azure/eventhub/_async/__init__.py @@ -7,6 +7,10 @@ import asyncio import time import datetime +try: + from urllib import urlparse, unquote_plus, urlencode, quote_plus +except ImportError: + from urllib.parse import urlparse, unquote_plus, urlencode, quote_plus from uamqp import authentication, constants, types, errors from uamqp import ( @@ -37,7 +41,7 @@ class EventHubClientAsync(EventHubClient): sending events to and receiving events from the Azure Event Hubs service. """ - def _create_auth(self, auth_uri, username, password): # pylint: disable=no-self-use + def _create_auth(self, username=None, password=None): # pylint: disable=no-self-use """ Create an ~uamqp.authentication.cbs_auth_async.SASTokenAuthAsync instance to authenticate the session. @@ -49,9 +53,11 @@ def _create_auth(self, auth_uri, username, password): # pylint: disable=no-self :param password: The shared access key. :type password: str """ + username = username or self._auth_config['username'] + password = password or self._auth_config['password'] if "@sas.root" in username: return authentication.SASLPlain(self.address.hostname, username, password) - return authentication.SASTokenAsync.from_shared_access_key(auth_uri, username, password, timeout=60) + return authentication.SASTokenAsync.from_shared_access_key(self.auth_uri, username, password, timeout=60) def _create_connection_async(self): """ @@ -108,11 +114,11 @@ async def _handle_redirect(self, redirects): redirects = [c.redirected for c in self.clients if c.redirected] if not all(r.hostname == redirects[0].hostname for r in redirects): raise EventHubError("Multiple clients attempting to redirect to different hosts.") - self.auth = self._create_auth(redirects[0].address.decode('utf-8'), **self._auth_config) - #port = str(redirects[0].port).encode('UTF-8') - #path = self.address.path.encode('UTF-8') - #self.mgmt_node = b"pyot/$management" #+ redirects[0].hostname b"amqps://pyot.azure-devices.net" + b":" + port + - #print("setting mgmt node", self.mgmt_node) + self.auth_uri = redirects[0].address.decode('utf-8') + self.auth = self._create_auth() + new_target, _, _ = self.auth_uri.partition("/ConsumerGroups") + self.address = urlparse(new_target) + self.mgmt_node = new_target.encode('UTF-8') + b"/$management" await self.connection.redirect_async(redirects[0], self.auth) await asyncio.gather(*[c.open_async(self.connection) for c in self.clients]) @@ -165,12 +171,12 @@ async def get_eventhub_info_async(self): :rtype: dict """ - self._create_connection_async() eh_name = self.address.path.lstrip('/') target = "amqps://{}/{}".format(self.address.hostname, eh_name) try: - mgmt_client = AMQPClientAsync(target, auth=self.auth, debug=self.debug) - await mgmt_client.open_async(connection=self.connection) + mgmt_auth = self._create_auth() + mgmt_client = AMQPClientAsync(target, auth=mgmt_auth, debug=self.debug) + await mgmt_client.open_async() mgmt_msg = Message(application_properties={'name': eh_name}) response = await mgmt_client.mgmt_request_async( mgmt_msg, diff --git a/azure/eventhub/client.py b/azure/eventhub/client.py index 8a47a03..d85df56 100644 --- a/azure/eventhub/client.py +++ b/azure/eventhub/client.py @@ -115,9 +115,9 @@ def __init__(self, address, username=None, password=None, debug=False): password = password or url_password if not username or not password: raise ValueError("Missing username and/or password.") - auth_uri = "sb://{}{}".format(self.address.hostname, self.address.path) - self.auth = self._create_auth(auth_uri, username, password) - self._auth_config = None + self.auth_uri = "sb://{}{}".format(self.address.hostname, self.address.path) + self._auth_config = {'username': username, 'password': password} + self.auth = self._create_auth() self.connection = None self.debug = debug @@ -147,11 +147,14 @@ def from_iothub_connection_string(cls, conn_str, **kwargs): username = "{}@sas.root.{}".format(policy, hub_name) password = _generate_sas_token(address, policy, key) client = cls("amqps://" + address, username=username, password=password, **kwargs) - client._auth_config = {'username': policy, 'password': key} # pylint: disable=protected-access - client.mgmt_node = ("amqps://" + address + ":5671/pyot/$management").encode('UTF-8') + client._auth_config = { + 'username': policy, + 'password': key, + 'iot_username': username, + 'iot_password': password} # pylint: disable=protected-access return client - def _create_auth(self, auth_uri, username, password): # pylint: disable=no-self-use + def _create_auth(self, username=None, password=None): """ Create an ~uamqp.authentication.SASTokenAuth instance to authenticate the session. @@ -163,9 +166,11 @@ def _create_auth(self, auth_uri, username, password): # pylint: disable=no-self :param password: The shared access key. :type password: str """ + username = username or self._auth_config['username'] + password = password or self._auth_config['password'] if "@sas.root" in username: return authentication.SASLPlain(self.address.hostname, username, password) - return authentication.SASTokenAuth.from_shared_access_key(auth_uri, username, password, timeout=60) + return authentication.SASTokenAuth.from_shared_access_key(self.auth_uri, username, password, timeout=60) def _create_properties(self): # pylint: disable=no-self-use """ @@ -225,7 +230,11 @@ def _handle_redirect(self, redirects): raise EventHubError("Some clients are attempting to redirect the connection.") if not all(r.hostname == redirects[0].hostname for r in redirects): raise EventHubError("Multiple clients attempting to redirect to different hosts.") - self.auth = self._create_auth(redirects[0].address.decode('utf-8'), **self._auth_config) + self.auth_uri = redirects[0].address.decode('utf-8') + self.auth = self._create_auth() + new_target, _, _ = self.auth_uri.partition("/ConsumerGroups") + self.address = urlparse(new_target) + self.mgmt_node = new_target.encode('UTF-8') + b"/$management" self.connection.redirect(redirects[0], self.auth) for client in self.clients: client.open(self.connection) @@ -284,12 +293,12 @@ def get_eventhub_info(self): :rtype: dict """ - self._create_connection() eh_name = self.address.path.lstrip('/') target = "amqps://{}/{}".format(self.address.hostname, eh_name) - mgmt_client = uamqp.AMQPClient(target, auth=self.auth, debug=self.debug) - mgmt_client.open(self.connection) + mgmt_auth = self._create_auth() + mgmt_client = uamqp.AMQPClient(target, auth=mgmt_auth, debug=self.debug) try: + mgmt_client.open() mgmt_msg = Message(application_properties={'name': eh_name}) response = mgmt_client.mgmt_request( mgmt_msg, diff --git a/azure/eventhub/common.py b/azure/eventhub/common.py index 9c3c2f4..f14e778 100644 --- a/azure/eventhub/common.py +++ b/azure/eventhub/common.py @@ -28,6 +28,8 @@ def _error_handler(error): return errors.ErrorAction(retry=True, backoff=2) elif error.condition == b'com.microsoft:operation-cancelled': return errors.ErrorAction(retry=True) + elif error.condition == b"com.microsoft:container-close": + return errors.ErrorAction(retry=True) return errors.ErrorAction(retry=True) diff --git a/azure/eventhub/sender.py b/azure/eventhub/sender.py index 83a6c4c..f635da8 100644 --- a/azure/eventhub/sender.py +++ b/azure/eventhub/sender.py @@ -27,6 +27,7 @@ def __init__(self, client, target, partition=None): :param target: The URI of the EventHub to send to. :type target: str """ + self.conneciton = None self.redirected = None self.error = None self.debug = client.debug @@ -52,6 +53,7 @@ def open(self, connection): :param connection: The underlying client shared connection. :type: connection: ~uamqp.connection.Connection """ + self.connection = connection if self.redirected: self._handler = SendClient( self.redirected.address, @@ -61,6 +63,22 @@ def open(self, connection): retry_policy=self.retry_policy) self._handler.open(connection) + def reconnect(self): + """If the Sender was disconnected from the service with + a retryable error - attempt to reconnect.""" + pending_states = (constants.MessageState.WaitingForSendAck, constants.MessageState.WaitingToBeSent) + unsent_events = [e for e in self._handler._pending_messages if e.state in pending_states] + self._handler.close() + self._handler = SendClient( + self.redirected.address, + auth=None, + debug=self.debug, + msg_timeout=Sender.TIMEOUT, + retry_policy=self.retry_policy) + self._handler.open(self.connection) + self._handler._pending_messages = unsent_events + self._handler.wait() + def get_handler_state(self): """ Get the state of the underlying handler with regards to start @@ -141,9 +159,12 @@ def send(self, event_data): self.close(exception=error) raise error except errors.LinkDetach as detach: - error = EventHubError(str(detach), detach) - self.close(exception=error) - raise error + if detach.action.retry: + self.reconnect() + else: + error = EventHubError(str(detach), detach) + self.close(exception=error) + raise error except errors.ConnectionClose as close: error = EventHubError(str(close), close) self.close(exception=error) @@ -182,9 +203,12 @@ def wait(self): try: self._handler.wait() except errors.LinkDetach as detach: - error = EventHubError(str(detach), detach) - self.close(exception=error) - raise error + if detach.action.retry: + self.reconnect() + else: + error = EventHubError(str(detach), detach) + self.close(exception=error) + raise error except errors.ConnectionClose as close: error = EventHubError(str(close), close) self.close(exception=error) diff --git a/conftest.py b/conftest.py index 6932e27..5fcf292 100644 --- a/conftest.py +++ b/conftest.py @@ -79,7 +79,7 @@ def invalid_policy(): @pytest.fixture() def iot_connection_str(): try: - return os.environ['IOT_HUB_CONNECTION_STR'] + return os.environ['IOTHUB_CONNECTION_STR'] except KeyError: pytest.skip("No IotHub connection string found.") diff --git a/tests/__init__.py b/tests/__init__.py index 7b7c91a..7ec7d3b 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -12,7 +12,7 @@ def get_logger(filename, level=logging.INFO): azure_logger = logging.getLogger("azure") azure_logger.setLevel(level) uamqp_logger = logging.getLogger("uamqp") - uamqp_logger.setLevel(logging.INFO) + uamqp_logger.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s') console_handler = logging.StreamHandler(stream=sys.stdout) diff --git a/tests/test_iothub_receive_async.py b/tests/test_iothub_receive_async.py index a7126d3..e1e718a 100644 --- a/tests/test_iothub_receive_async.py +++ b/tests/test_iothub_receive_async.py @@ -25,13 +25,14 @@ async def pump(receiver, sleep=None): @pytest.mark.asyncio -async def test_iothub_receive_async(iot_connection_str): +async def test_iothub_receive_multiple_async(iot_connection_str): client = EventHubClientAsync.from_iothub_connection_string(iot_connection_str, debug=True) - receivers = [] - for i in range(2): - receivers.append(client.add_async_receiver("$default", "0", prefetch=1000, operation='/messages/events')) - await client.run_async() try: + receivers = [] + for i in range(2): + receivers.append(client.add_async_receiver("$default", "0", prefetch=1000, operation='/messages/events')) + await client.run_async() + partitions = await client.get_eventhub_info_async() outputs = await asyncio.gather( pump(receivers[0]), pump(receivers[1]), @@ -39,8 +40,6 @@ async def test_iothub_receive_async(iot_connection_str): assert isinstance(outputs[0], int) and outputs[0] == 0 assert isinstance(outputs[1], int) and outputs[1] == 0 - except: - raise finally: await client.stop_async() @@ -60,7 +59,5 @@ async def test_iothub_receive_detach_async(iot_connection_str): assert isinstance(outputs[0], int) and outputs[0] == 0 assert isinstance(outputs[1], EventHubError) - except: - raise finally: await client.stop_async() \ No newline at end of file diff --git a/tests/test_iothub_send.py b/tests/test_iothub_send.py index 7c0dd7c..13c4eef 100644 --- a/tests/test_iothub_send.py +++ b/tests/test_iothub_send.py @@ -19,6 +19,8 @@ def test_iothub_send_single_event(iot_connection_str, device_id): client = EventHubClient.from_iothub_connection_string(iot_connection_str, debug=True) sender = client.add_sender(operation='/messages/devicebound') try: + with pytest.raises(NotImplementedError): + partitions = client.get_eventhub_info() client.run() outcome = sender.send(EventData(b"A single event", to_device=device_id)) assert outcome.value == 0 diff --git a/tests/test_negative.py b/tests/test_negative.py index 753f2e2..fc41287 100644 --- a/tests/test_negative.py +++ b/tests/test_negative.py @@ -221,7 +221,7 @@ async def test_send_to_invalid_partitions_async(connection_str): def test_send_too_large_message(connection_str): - client = EventHubClient.from_connection_string(connection_str, debug=False) + client = EventHubClient.from_connection_string(connection_str, debug=True) sender = client.add_sender() try: client.run() @@ -299,6 +299,8 @@ async def test_max_receivers_async(connection_str, senders): pump(receivers[5]), return_exceptions=True) print(outputs) - assert len([o for o in outputs if isinstance(o, EventHubError)]) == 1 + failed = [o for o in outputs if isinstance(o, EventHubError)] + assert len(failed) == 1 + print(failed[0].message) finally: await client.stop_async() \ No newline at end of file diff --git a/tests/test_receive_async.py b/tests/test_receive_async.py index d002674..8cfb370 100644 --- a/tests/test_receive_async.py +++ b/tests/test_receive_async.py @@ -216,11 +216,13 @@ async def test_epoch_receiver_async(connection_str, senders): @pytest.mark.asyncio async def test_multiple_receiver_async(connection_str, senders): client = EventHubClientAsync.from_connection_string(connection_str, debug=True) + partitions = await client.get_eventhub_info_async() receivers = [] for i in range(2): receivers.append(client.add_async_receiver("$default", "0", prefetch=10)) try: await client.run_async() + more_partitions = await client.get_eventhub_info_async() outputs = await asyncio.gather( pump(receivers[0]), pump(receivers[1]), From d98daa464fc153e97117e596bf401bd9958f66f4 Mon Sep 17 00:00:00 2001 From: annatisch Date: Thu, 26 Jul 2018 11:51:06 -0700 Subject: [PATCH 17/52] Some test updates --- tests/test_iothub_receive.py | 2 +- tests/test_iothub_receive_async.py | 37 +++----- tests/test_iothub_send.py | 3 +- tests/test_longrunning_receive.py | 2 +- tests/test_longrunning_send_async.py | 111 +++++++++++++++++++++++ tests/test_negative.py | 2 +- tests/test_receive.py | 11 ++- tests/test_receive_async.py | 6 +- tests/test_reconnect.py | 128 +++++++++++++++++++++++++++ tests/test_send.py | 4 +- 10 files changed, 269 insertions(+), 37 deletions(-) create mode 100644 tests/test_longrunning_send_async.py create mode 100644 tests/test_reconnect.py diff --git a/tests/test_iothub_receive.py b/tests/test_iothub_receive.py index 78c1de8..a48274b 100644 --- a/tests/test_iothub_receive.py +++ b/tests/test_iothub_receive.py @@ -11,7 +11,7 @@ from azure import eventhub from azure.eventhub import EventData, EventHubClient, Offset -def test_iothub_receive(iot_connection_str, device_id): +def test_iothub_receive_sync(iot_connection_str, device_id): client = EventHubClient.from_iothub_connection_string(iot_connection_str, debug=True) receiver = client.add_receiver("$default", "0", operation='/messages/events') try: diff --git a/tests/test_iothub_receive_async.py b/tests/test_iothub_receive_async.py index e1e718a..d26c00c 100644 --- a/tests/test_iothub_receive_async.py +++ b/tests/test_iothub_receive_async.py @@ -24,40 +24,29 @@ async def pump(receiver, sleep=None): return messages -@pytest.mark.asyncio -async def test_iothub_receive_multiple_async(iot_connection_str): - client = EventHubClientAsync.from_iothub_connection_string(iot_connection_str, debug=True) +async def get_partitions(iot_connection_str): try: - receivers = [] - for i in range(2): - receivers.append(client.add_async_receiver("$default", "0", prefetch=1000, operation='/messages/events')) + client = EventHubClientAsync.from_iothub_connection_string(iot_connection_str, debug=True) + client.add_async_receiver("$default", "0", prefetch=1000, operation='/messages/events') await client.run_async() partitions = await client.get_eventhub_info_async() - outputs = await asyncio.gather( - pump(receivers[0]), - pump(receivers[1]), - return_exceptions=True) - - assert isinstance(outputs[0], int) and outputs[0] == 0 - assert isinstance(outputs[1], int) and outputs[1] == 0 + return partitions["partition_ids"] finally: await client.stop_async() @pytest.mark.asyncio -async def test_iothub_receive_detach_async(iot_connection_str): +async def test_iothub_receive_multiple_async(iot_connection_str): + partitions = await get_partitions(iot_connection_str) client = EventHubClientAsync.from_iothub_connection_string(iot_connection_str, debug=True) - receivers = [] - for i in range(2): - receivers.append(client.add_async_receiver("$default", str(i), prefetch=1000, operation='/messages/events')) - await client.run_async() try: - outputs = await asyncio.gather( - pump(receivers[0]), - pump(receivers[1]), - return_exceptions=True) + receivers = [] + for p in partitions: + receivers.append(client.add_async_receiver("$default", p, prefetch=1000, operation='/messages/events')) + await client.run_async() + outputs = await asyncio.gather(*[pump(r) for r in receivers]) assert isinstance(outputs[0], int) and outputs[0] == 0 - assert isinstance(outputs[1], EventHubError) + assert isinstance(outputs[1], int) and outputs[1] == 0 finally: - await client.stop_async() \ No newline at end of file + await client.stop_async() diff --git a/tests/test_iothub_send.py b/tests/test_iothub_send.py index 13c4eef..3f39c61 100644 --- a/tests/test_iothub_send.py +++ b/tests/test_iothub_send.py @@ -19,9 +19,8 @@ def test_iothub_send_single_event(iot_connection_str, device_id): client = EventHubClient.from_iothub_connection_string(iot_connection_str, debug=True) sender = client.add_sender(operation='/messages/devicebound') try: - with pytest.raises(NotImplementedError): - partitions = client.get_eventhub_info() client.run() + partitions = client.get_eventhub_info() outcome = sender.send(EventData(b"A single event", to_device=device_id)) assert outcome.value == 0 except: diff --git a/tests/test_longrunning_receive.py b/tests/test_longrunning_receive.py index 60007a5..b32731b 100644 --- a/tests/test_longrunning_receive.py +++ b/tests/test_longrunning_receive.py @@ -48,7 +48,7 @@ async def pump(_pid, receiver, _args, _dl): _pid, total, batch[-1].sequence_number, - batch[-1].offset)) + batch[-1].offset.value)) print("{}: total received {}".format( _pid, total)) diff --git a/tests/test_longrunning_send_async.py b/tests/test_longrunning_send_async.py new file mode 100644 index 0000000..afc13fa --- /dev/null +++ b/tests/test_longrunning_send_async.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python + +""" +send test +""" + +import logging +import argparse +import time +import threading +import os +import asyncio + +from azure.eventhub import EventHubClientAsync, EventData + +try: + import tests + logger = tests.get_logger("send_test.log", logging.INFO) +except ImportError: + logger = logging.getLogger("uamqp") + logger.setLevel(logging.INFO) + + +def check_send_successful(outcome, condition): + if outcome.value != 0: + print("Send failed {}".format(condition)) + + +async def get_partitions(args): + #client = EventHubClientAsync.from_connection_string( + # args.conn_str, + # eventhub=args.eventhub, debug=True) + eh_data = await args.get_eventhub_info_async() + return eh_data["partition_ids"] + + +async def pump(pid, sender, args, duration): + deadline = time.time() + duration + total = 0 + + def data_generator(): + for i in range(args.batch): + yield b"D" * args.payload + + if args.batch > 1: + logger.error("Sending batched messages") + else: + logger.error("Sending single messages") + + try: + while time.time() < deadline: + if args.batch > 1: + data = EventData(batch=data_generator()) + else: + data = EventData(body=b"D" * args.payload) + sender.transfer(data, callback=check_send_successful) + total += args.batch + if total % 10000 == 0: + await sender.wait_async() + logger.error("Send total {}".format(total)) + except Exception as err: + logger.error("Send failed {}".format(err)) + logger.error("Sent total {}".format(total)) + + +def test_long_running_partition_send_async(): + parser = argparse.ArgumentParser() + parser.add_argument("--duration", help="Duration in seconds of the test", type=int, default=30) + parser.add_argument("--payload", help="payload size", type=int, default=512) + parser.add_argument("--batch", help="Number of events to send and wait", type=int, default=1) + parser.add_argument("--partitions", help="Comma seperated partition IDs") + parser.add_argument("--conn-str", help="EventHub connection string", default=os.environ.get('EVENT_HUB_CONNECTION_STR')) + parser.add_argument("--eventhub", help="Name of EventHub") + parser.add_argument("--address", help="Address URI to the EventHub entity") + parser.add_argument("--sas-policy", help="Name of the shared access policy to authenticate with") + parser.add_argument("--sas-key", help="Shared access key") + + loop = asyncio.get_event_loop() + args, _ = parser.parse_known_args() + if args.conn_str: + client = EventHubClientAsync.from_connection_string( + args.conn_str, + eventhub=args.eventhub, debug=True) + elif args.address: + client = EventHubClient( + args.address, + username=args.sas_policy, + password=args.sas_key) + else: + try: + import pytest + pytest.skip("Must specify either '--conn-str' or '--address'") + except ImportError: + raise ValueError("Must specify either '--conn-str' or '--address'") + + try: + if not args.partitions: + partitions = loop.run_until_complete(get_partitions(client)) + else: + partitions = args.partitions.split(",") + pumps = [] + for pid in partitions: + sender = client.add_async_sender(partition=pid) + pumps.append(pump(pid, sender, args, args.duration)) + loop.run_until_complete(client.run_async()) + loop.run_until_complete(asyncio.gather(*pumps)) + finally: + loop.run_until_complete(client.stop_async()) + +if __name__ == '__main__': + test_long_running_partition_send_async() diff --git a/tests/test_negative.py b/tests/test_negative.py index fc41287..dbc8096 100644 --- a/tests/test_negative.py +++ b/tests/test_negative.py @@ -181,7 +181,7 @@ def test_receive_from_invalid_partitions_sync(connection_str): async def test_receive_from_invalid_partitions_async(connection_str): partitions = ["XYZ", "-1", "1000", "-" ] for p in partitions: - client = EventHubClientAsync.from_connection_string(connection_str, debug=False) + client = EventHubClientAsync.from_connection_string(connection_str, debug=True) receiver = client.add_async_receiver("$default", p) try: with pytest.raises(EventHubError): diff --git a/tests/test_receive.py b/tests/test_receive.py index 44cb7b2..fda5a96 100644 --- a/tests/test_receive.py +++ b/tests/test_receive.py @@ -31,7 +31,7 @@ def test_receive_end_of_stream(connection_str, senders): client.stop() -def test_receive_with_offset(connection_str, senders): +def test_receive_with_offset_sync(connection_str, senders): client = EventHubClient.from_connection_string(connection_str, debug=False) receiver = client.add_receiver("$default", "0", offset=Offset('@latest')) try: @@ -44,7 +44,7 @@ def test_receive_with_offset(connection_str, senders): assert len(received) == 1 offset = received[0].offset - offset_receiver = client.add_receiver("$default", "0", offset=Offset(offset)) + offset_receiver = client.add_receiver("$default", "0", offset=offset) client.run() received = offset_receiver.receive(timeout=5) assert len(received) == 0 @@ -71,7 +71,7 @@ def test_receive_with_inclusive_offset(connection_str, senders): assert len(received) == 1 offset = received[0].offset - offset_receiver = client.add_receiver("$default", "0", offset=Offset(offset, inclusive=True)) + offset_receiver = client.add_receiver("$default", "0", offset=Offset(offset.value, inclusive=True)) client.run() received = offset_receiver.receive(timeout=5) assert len(received) == 1 @@ -83,10 +83,13 @@ def test_receive_with_inclusive_offset(connection_str, senders): def test_receive_with_datetime(connection_str, senders): client = EventHubClient.from_connection_string(connection_str, debug=False) + partitions = client.get_eventhub_info() + assert partitions["partition_ids"] == ["0", "1"] receiver = client.add_receiver("$default", "0", offset=Offset('@latest')) try: client.run() - + more_partitions = client.get_eventhub_info() + assert more_partitions["partition_ids"] == ["0", "1"] received = receiver.receive(timeout=5) assert len(received) == 0 senders[0].send(EventData(b"Data")) diff --git a/tests/test_receive_async.py b/tests/test_receive_async.py index 8cfb370..267e82c 100644 --- a/tests/test_receive_async.py +++ b/tests/test_receive_async.py @@ -46,7 +46,7 @@ async def test_receive_with_offset_async(connection_str, senders): assert len(received) == 1 offset = received[0].offset - offset_receiver = client.add_async_receiver("$default", "0", offset=Offset(offset)) + offset_receiver = client.add_async_receiver("$default", "0", offset=offset) await client.run_async() received = await offset_receiver.receive(timeout=5) assert len(received) == 0 @@ -73,7 +73,7 @@ async def test_receive_with_inclusive_offset_async(connection_str, senders): assert len(received) == 1 offset = received[0].offset - offset_receiver = client.add_async_receiver("$default", "0", offset=Offset(offset, inclusive=True)) + offset_receiver = client.add_async_receiver("$default", "0", offset=Offset(offset.value, inclusive=True)) await client.run_async() received = await offset_receiver.receive(timeout=5) assert len(received) == 1 @@ -217,12 +217,14 @@ async def test_epoch_receiver_async(connection_str, senders): async def test_multiple_receiver_async(connection_str, senders): client = EventHubClientAsync.from_connection_string(connection_str, debug=True) partitions = await client.get_eventhub_info_async() + assert partitions["partition_ids"] == ["0", "1"] receivers = [] for i in range(2): receivers.append(client.add_async_receiver("$default", "0", prefetch=10)) try: await client.run_async() more_partitions = await client.get_eventhub_info_async() + assert more_partitions["partition_ids"] == ["0", "1"] outputs = await asyncio.gather( pump(receivers[0]), pump(receivers[1]), diff --git a/tests/test_reconnect.py b/tests/test_reconnect.py new file mode 100644 index 0000000..a6aa0ce --- /dev/null +++ b/tests/test_reconnect.py @@ -0,0 +1,128 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +#-------------------------------------------------------------------------- + +import os +import time +import asyncio +import pytest + +from azure import eventhub +from azure.eventhub import ( + EventHubClientAsync, + EventData, + Offset, + EventHubError, + EventHubClient) + + +def test_send_with_long_interval_sync(connection_str, receivers): + #pytest.skip("long running") + client = EventHubClient.from_connection_string(connection_str, debug=True) + sender = client.add_sender() + try: + client.run() + sender.send(EventData(b"A single event")) + for _ in range(2): + time.sleep(300) + sender.send(EventData(b"A single event")) + finally: + client.stop() + + received = [] + for r in receivers: + received.extend(r.receive(timeout=1)) + + assert len(received) == 3 + assert list(received[0].body)[0] == b"A single event" + + +@pytest.mark.asyncio +async def test_send_with_long_interval_async(connection_str, receivers): + #pytest.skip("long running") + client = EventHubClientAsync.from_connection_string(connection_str, debug=True) + sender = client.add_async_sender() + try: + await client.run_async() + await sender.send(EventData(b"A single event")) + for _ in range(2): + await asyncio.sleep(300) + await sender.send(EventData(b"A single event")) + finally: + await client.stop_async() + + received = [] + for r in receivers: + received.extend(r.receive(timeout=1)) + assert len(received) == 3 + assert list(received[0].body)[0] == b"A single event" + + +def test_send_with_forced_conn_close_sync(connection_str, receivers): + #pytest.skip("long running") + client = EventHubClient.from_connection_string(connection_str, debug=True) + sender = client.add_sender() + try: + client.run() + sender.send(EventData(b"A single event")) + sender._handler._message_sender.destroy() + time.sleep(300) + sender.send(EventData(b"A single event")) + sender.send(EventData(b"A single event")) + sender._handler._message_sender.destroy() + time.sleep(300) + sender.send(EventData(b"A single event")) + sender.send(EventData(b"A single event")) + finally: + client.stop() + + received = [] + for r in receivers: + received.extend(r.receive(timeout=1)) + assert len(received) == 5 + assert list(received[0].body)[0] == b"A single event" + + +@pytest.mark.asyncio +async def test_send_with_forced_conn_close_async(connection_str, receivers): + #pytest.skip("long running") + client = EventHubClientAsync.from_connection_string(connection_str, debug=True) + sender = client.add_async_sender() + try: + await client.run_async() + await sender.send(EventData(b"A single event")) + sender._handler._message_sender.destroy() + await asyncio.sleep(300) + await sender.send(EventData(b"A single event")) + await sender.send(EventData(b"A single event")) + sender._handler._message_sender.destroy() + await asyncio.sleep(300) + await sender.send(EventData(b"A single event")) + await sender.send(EventData(b"A single event")) + finally: + await client.stop_async() + + received = [] + for r in receivers: + received.extend(r.receive(timeout=1)) + assert len(received) == 5 + assert list(received[0].body)[0] == b"A single event" + + +# def test_send_with_forced_link_detach(connection_str, receivers): +# client = EventHubClient.from_connection_string(connection_str, debug=True) +# sender = client.add_sender() +# size = 20 * 1024 +# try: +# client.run() +# for i in range(1000): +# sender.transfer(EventData([b"A"*size, b"B"*size, b"C"*size, b"D"*size, b"A"*size, b"B"*size, b"C"*size, b"D"*size, b"A"*size, b"B"*size, b"C"*size, b"D"*size])) +# sender.wait() +# finally: +# client.stop() + +# received = [] +# for r in receivers: +# received.extend(r.receive(timeout=10)) diff --git a/tests/test_send.py b/tests/test_send.py index faf116f..a74c3d1 100644 --- a/tests/test_send.py +++ b/tests/test_send.py @@ -164,8 +164,8 @@ def batched(): assert len(partition_1) == 10 -def test_send_array(connection_str, receivers): - client = EventHubClient.from_connection_string(connection_str, debug=False) +def test_send_array_sync(connection_str, receivers): + client = EventHubClient.from_connection_string(connection_str, debug=True) sender = client.add_sender() try: client.run() From 1ba74a50f0182db9e8aff57db1bdd2ba32f9e03f Mon Sep 17 00:00:00 2001 From: annatisch Date: Thu, 26 Jul 2018 11:53:28 -0700 Subject: [PATCH 18/52] Updated uamqp dependency --- conftest.py | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conftest.py b/conftest.py index 5fcf292..96d135d 100644 --- a/conftest.py +++ b/conftest.py @@ -94,7 +94,7 @@ def device_id(): @pytest.fixture() def receivers(connection_str): - client = EventHubClient.from_connection_string(connection_str, debug=True) + client = EventHubClient.from_connection_string(connection_str, debug=False) eh_hub_info = client.get_eventhub_info() partitions = eh_hub_info["partition_ids"] @@ -114,7 +114,7 @@ def receivers(connection_str): @pytest.fixture() def senders(connection_str): - client = EventHubClient.from_connection_string(connection_str, debug=False) + client = EventHubClient.from_connection_string(connection_str, debug=True) eh_hub_info = client.get_eventhub_info() partitions = eh_hub_info["partition_ids"] diff --git a/setup.py b/setup.py index 9fce5a2..df46435 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ zip_safe=False, packages=find_packages(exclude=["examples", "tests"]), install_requires=[ - 'uamqp~=0.1.0', + 'uamqp~=0.2.0', 'msrestazure~=0.4.11', 'azure-common~=1.1', 'azure-storage~=0.36.0' From 62c8e83c90a949ee52a0e623115b0dcfd2aa55e2 Mon Sep 17 00:00:00 2001 From: annatisch Date: Thu, 26 Jul 2018 12:09:08 -0700 Subject: [PATCH 19/52] Restructure for independent connections --- azure/eventhub/_async/__init__.py | 62 +++----------- azure/eventhub/_async/receiver_async.py | 91 ++++++++++++++++----- azure/eventhub/_async/sender_async.py | 98 +++++++++++++++------- azure/eventhub/client.py | 80 +++++++----------- azure/eventhub/common.py | 13 ++- azure/eventhub/receiver.py | 103 ++++++++++++++++-------- azure/eventhub/sender.py | 78 ++++++++++-------- 7 files changed, 304 insertions(+), 221 deletions(-) diff --git a/azure/eventhub/_async/__init__.py b/azure/eventhub/_async/__init__.py index e961302..3041da5 100644 --- a/azure/eventhub/_async/__init__.py +++ b/azure/eventhub/_async/__init__.py @@ -15,7 +15,6 @@ from uamqp import authentication, constants, types, errors from uamqp import ( Message, - Source, ConnectionAsync, AMQPClientAsync, SendClientAsync, @@ -59,29 +58,6 @@ def _create_auth(self, username=None, password=None): # pylint: disable=no-self return authentication.SASLPlain(self.address.hostname, username, password) return authentication.SASTokenAsync.from_shared_access_key(self.auth_uri, username, password, timeout=60) - def _create_connection_async(self): - """ - Create a new ~uamqp._async.connection_async.ConnectionAsync instance that will be shared between all - AsyncSender/AsyncReceiver clients. - """ - if not self.connection: - log.info("{}: Creating connection with address={}".format( - self.container_id, self.address.geturl())) - self.connection = ConnectionAsync( - self.address.hostname, - self.auth, - container_id=self.container_id, - properties=self._create_properties(), - debug=self.debug) - - async def _close_connection_async(self): - """ - Close and destroy the connection async. - """ - if self.connection: - await self.connection.destroy_async() - self.connection = None - async def _close_clients_async(self): """ Close all open AsyncSender/AsyncReceiver clients. @@ -91,17 +67,13 @@ async def _close_clients_async(self): async def _wait_for_client(self, client): try: while client.get_handler_state().value == 2: - await self.connection.work_async() + await client._handler._connection.work_async() except Exception as exp: # pylint: disable=broad-except await client.close_async(exception=exp) async def _start_client_async(self, client): try: - await client.open_async(self.connection) - started = await client.has_started() - while not started: - await self.connection.work_async() - started = await client.has_started() + await client.open_async() except Exception as exp: # pylint: disable=broad-except await client.close_async(exception=exp) @@ -114,13 +86,8 @@ async def _handle_redirect(self, redirects): redirects = [c.redirected for c in self.clients if c.redirected] if not all(r.hostname == redirects[0].hostname for r in redirects): raise EventHubError("Multiple clients attempting to redirect to different hosts.") - self.auth_uri = redirects[0].address.decode('utf-8') - self.auth = self._create_auth() - new_target, _, _ = self.auth_uri.partition("/ConsumerGroups") - self.address = urlparse(new_target) - self.mgmt_node = new_target.encode('UTF-8') + b"/$management" - await self.connection.redirect_async(redirects[0], self.auth) - await asyncio.gather(*[c.open_async(self.connection) for c in self.clients]) + self._process_redirect_uri(redirects[0]) + await asyncio.gather(*[c.open_async() for c in self.clients]) async def run_async(self): """ @@ -135,7 +102,6 @@ async def run_async(self): :rtype: list[~azure.eventhub.common.EventHubError] """ log.info("{}: Starting {} clients".format(self.container_id, len(self.clients))) - self._create_connection_async() tasks = [self._start_client_async(c) for c in self.clients] try: await asyncio.gather(*tasks) @@ -163,7 +129,6 @@ async def stop_async(self): log.info("{}: Stopping {} clients".format(self.container_id, len(self.clients))) self.stopped = True await self._close_clients_async() - await self._close_connection_async() async def get_eventhub_info_async(self): """ @@ -171,18 +136,18 @@ async def get_eventhub_info_async(self): :rtype: dict """ - eh_name = self.address.path.lstrip('/') - target = "amqps://{}/{}".format(self.address.hostname, eh_name) + alt_creds = { + "username": self._auth_config.get("iot_username"), + "password":self._auth_config.get("iot_password")} try: - mgmt_auth = self._create_auth() - mgmt_client = AMQPClientAsync(target, auth=mgmt_auth, debug=self.debug) + mgmt_auth = self._create_auth(**alt_creds) + mgmt_client = AMQPClientAsync(self.mgmt_target, auth=mgmt_auth, debug=self.debug) await mgmt_client.open_async() - mgmt_msg = Message(application_properties={'name': eh_name}) + mgmt_msg = Message(application_properties={'name': self.eh_name}) response = await mgmt_client.mgmt_request_async( mgmt_msg, constants.READ_OPERATION, op_type=b'com.microsoft:eventhub', - node=self.mgmt_node, status_code_field=b'status-code', description_fields=b'status-description') eh_info = response.get_data() @@ -215,12 +180,11 @@ def add_async_receiver(self, consumer_group, partition, offset=None, prefetch=30 :rtype: ~azure.eventhub._async.receiver_async.ReceiverAsync """ path = self.address.path + operation if operation else self.address.path + source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( self.address.hostname, path, consumer_group, partition) - source = Source(source_url) - if offset is not None: - source.set_filter(offset.selector()) - handler = AsyncReceiver(self, source, prefetch=prefetch, loop=loop) + print("RECEIVER_PATH", source_url) + handler = AsyncReceiver(self, source_url, offset=offset, prefetch=prefetch, loop=loop) self.clients.append(handler) return handler diff --git a/azure/eventhub/_async/receiver_async.py b/azure/eventhub/_async/receiver_async.py index b3b5138..072e04a 100644 --- a/azure/eventhub/_async/receiver_async.py +++ b/azure/eventhub/_async/receiver_async.py @@ -6,10 +6,11 @@ import asyncio from uamqp import errors, types -from uamqp import ReceiveClientAsync +from uamqp import ReceiveClientAsync, Source from azure.eventhub import EventHubError, EventData from azure.eventhub.receiver import Receiver +from azure.eventhub.common import _error_handler class AsyncReceiver(Receiver): @@ -17,7 +18,7 @@ class AsyncReceiver(Receiver): Implements the async API of a Receiver. """ - def __init__(self, client, source, prefetch=300, epoch=None, loop=None): # pylint: disable=super-init-not-called + def __init__(self, client, source, offset=None, prefetch=300, epoch=None, loop=None): # pylint: disable=super-init-not-called """ Instantiate an async receiver. @@ -33,25 +34,32 @@ def __init__(self, client, source, prefetch=300, epoch=None, loop=None): # pyli :param loop: An event loop. """ self.loop = loop or asyncio.get_event_loop() + self.client = client + self.source = source + self.offset = offset + self.prefetch = prefetch + self.epoch = epoch + self.retry_policy = errors.ErrorPolicy(max_retries=3, on_error=_error_handler) self.redirected = None self.error = None - self.debug = client.debug - self.offset = None - self.prefetch = prefetch self.properties = None - self.epoch = epoch + source = Source(self.source) + if self.offset is not None: + source.set_filter(self.offset.selector()) if epoch: self.properties = {types.AMQPSymbol(self._epoch): types.AMQPLong(int(epoch))} self._handler = ReceiveClientAsync( source, - auth=client.auth, - debug=self.debug, + auth=self.client.get_auth(), + debug=self.client.debug, prefetch=self.prefetch, link_properties=self.properties, timeout=self.timeout, + error_policy=self.retry_policy, + keep_alive_interval=30, loop=self.loop) - async def open_async(self, connection): + async def open_async(self): """ Open the Receiver using the supplied conneciton. If the handler has previously been redirected, the redirect @@ -61,15 +69,51 @@ async def open_async(self, connection): :type: connection: ~uamqp._async.connection_async.ConnectionAsync """ if self.redirected: + self.source = self.redirected.address + source = Source(self.source) + if self.offset is not None: + source.set_filter(self.offset.selector()) + alt_creds = { + "username": self.client._auth_config.get("iot_username"), + "password":self.client._auth_config.get("iot_password")} self._handler = ReceiveClientAsync( - self.redirected.address, - auth=None, - debug=self.debug, + source, + auth=self.client.get_auth(**alt_creds), + debug=self.client.debug, prefetch=self.prefetch, link_properties=self.properties, timeout=self.timeout, + error_policy=self.retry_policy, + keep_alive_interval=30, loop=self.loop) - await self._handler.open_async(connection=connection) + await self._handler.open_async() + while not await self.has_started(): + await self._handler._connection.work_async() + + async def reconnect_async(self): + """If the Receiver was disconnected from the service with + a retryable error - attempt to reconnect.""" + alt_creds = { + "username": self.client._auth_config.get("iot_username"), + "password":self.client._auth_config.get("iot_password")} + await self._handler.close_async() + source = Source(self.source) + if self.offset is not None: + source.set_filter(self.offset.selector()) + self._handler = ReceiveClientAsync( + source, + auth=self.client.get_auth(**alt_creds), + debug=self.client.debug, + prefetch=self.prefetch, + link_properties=self.properties, + timeout=self.timeout, + error_policy=self.retry_policy, + keep_alive_interval=30, + properties=self.client.create_properties(), + loop=self.loop) + await self._handler.open_async() + while not await self.has_started(): + await self._handler._connection.work_async() async def has_started(self): """ @@ -131,25 +175,28 @@ async def receive(self, max_batch_size=None, timeout=None): """ if self.error: raise self.error + data_batch = [] try: timeout_ms = 1000 * timeout if timeout else 0 message_batch = await self._handler.receive_message_batch_async( max_batch_size=max_batch_size, timeout=timeout_ms) - data_batch = [] for message in message_batch: event_data = EventData(message=message) self.offset = event_data.offset data_batch.append(event_data) return data_batch - except errors.LinkDetach as detach: - error = EventHubError(str(detach), detach) - await self.close_async(exception=error) - raise error - except errors.ConnectionClose as close: - error = EventHubError(str(close), close) - await self.close_async(exception=error) - raise error + except (errors.LinkDetach, errors.ConnectionClose) as shutdown: + if shutdown.action.retry: + await self.reconnect_async() + return data_batch + else: + error = EventHubError(str(shutdown), shutdown) + await self.close_async(exception=error) + raise error + except (errors.MessageHandlerError): + await self.reconnect_async() + return data_batch except Exception as e: error = EventHubError("Receive failed: {}".format(e)) await self.close_async(exception=error) diff --git a/azure/eventhub/_async/sender_async.py b/azure/eventhub/_async/sender_async.py index abcd0fa..3a10d6f 100644 --- a/azure/eventhub/_async/sender_async.py +++ b/azure/eventhub/_async/sender_async.py @@ -10,6 +10,7 @@ from azure.eventhub import EventHubError from azure.eventhub.sender import Sender +from azure.eventhub.common import _error_handler class AsyncSender(Sender): """ @@ -26,23 +27,28 @@ def __init__(self, client, target, partition=None, loop=None): # pylint: disabl :type target: str :param loop: An event loop. """ + self.loop = loop or asyncio.get_event_loop() + self.client = client + self.target = target + self.partition = partition + self.retry_policy = errors.ErrorPolicy(max_retries=3, on_error=_error_handler) self.redirected = None self.error = None - self.debug = client.debug - self.partition = partition if partition: - target += "/Partitions/" + partition - self.loop = loop or asyncio.get_event_loop() + self.target += "/Partitions/" + partition self._handler = SendClientAsync( - target, - auth=client.auth, - debug=self.debug, + self.target, + auth=self.client.get_auth(), + debug=self.client.debug, msg_timeout=Sender.TIMEOUT, + error_policy=self.retry_policy, + keep_alive_interval=30, + properties=self.client.create_properties(), loop=self.loop) self._outcome = None self._condition = None - async def open_async(self, connection): + async def open_async(self): """ Open the Sender using the supplied conneciton. If the handler has previously been redirected, the redirect @@ -52,12 +58,44 @@ async def open_async(self, connection): :type: connection:~uamqp._async.connection_async.ConnectionAsync """ if self.redirected: + self.target = self.redirected.address + alt_creds = { + "username": self.client._auth_config.get("iot_username"), + "password":self.client._auth_config.get("iot_password")} self._handler = SendClientAsync( - self.redirected.address, - auth=None, - debug=self.debug, - msg_timeout=Sender.TIMEOUT) - await self._handler.open_async(connection=connection) + self.target, + auth=self.client.get_auth(**alt_creds), + debug=self.client.debug, + msg_timeout=Sender.TIMEOUT, + error_policy=self.retry_policy, + keep_alive_interval=30, + properties=self.client.create_properties(), + loop=self.loop) + await self._handler.open_async() + while not await self.has_started(): + await self._handler._connection.work_async() + + async def reconnect_async(self): + """If the Receiver was disconnected from the service with + a retryable error - attempt to reconnect.""" + pending_states = (constants.MessageState.WaitingForSendAck, constants.MessageState.WaitingToBeSent) + unsent_events = [e for e in self._handler._pending_messages if e.state in pending_states] + alt_creds = { + "username": self.client._auth_config.get("iot_username"), + "password":self.client._auth_config.get("iot_password")} + await self._handler.close_async() + self._handler = SendClientAsync( + self.target, + auth=self.client.get_auth(**alt_creds), + debug=self.client.debug, + msg_timeout=Sender.TIMEOUT, + error_policy=self.retry_policy, + keep_alive_interval=30, + properties=self.client.create_properties(), + loop=self.loop) + await self._handler.open_async() + self._handler._pending_messages = unsent_events + await self._handler.wait_async() async def has_started(self): """ @@ -124,14 +162,15 @@ async def send(self, event_data): await self._handler.send_message_async(event_data.message) if self._outcome != constants.MessageSendResult.Ok: raise Sender._error(self._outcome, self._condition) - except errors.LinkDetach as detach: - error = EventHubError(str(detach), detach) - await self.close_async(exception=error) - raise error - except errors.ConnectionClose as close: - error = EventHubError(str(close), close) - await self.close_async(exception=error) - raise error + except (errors.LinkDetach, errors.ConnectionClose) as shutdown: + if shutdown.action.retry: + await self.reconnect_async() + else: + error = EventHubError(str(shutdown), shutdown) + await self.close_async(exception=error) + raise error + except (errors.MessageHandlerError): + await self.reconnect_async() except Exception as e: error = EventHubError("Send failed: {}".format(e)) await self.close_async(exception=error) @@ -147,13 +186,14 @@ async def wait_async(self): raise self.error try: await self._handler.wait_async() - except errors.LinkDetach as detach: - error = EventHubError(str(detach), detach) - await self.close_async(exception=error) - raise error - except errors.ConnectionClose as close: - error = EventHubError(str(close), close) - await self.close_async(exception=error) - raise error + except (errors.LinkDetach, errors.ConnectionClose) as shutdown: + if shutdown.action.retry: + await self.reconnect_async() + else: + error = EventHubError(str(shutdown), shutdown) + await self.close_async(exception=error) + raise error + except (errors.MessageHandlerError): + await self.reconnect_async() except Exception as e: raise EventHubError("Send failed: {}".format(e)) diff --git a/azure/eventhub/client.py b/azure/eventhub/client.py index d85df56..1659f70 100644 --- a/azure/eventhub/client.py +++ b/azure/eventhub/client.py @@ -8,6 +8,7 @@ import sys import uuid import time +import functools try: from urllib import urlparse, unquote_plus, urlencode, quote_plus except ImportError: @@ -16,7 +17,6 @@ import uamqp from uamqp import Connection from uamqp import Message -from uamqp import Source from uamqp import authentication from uamqp import constants @@ -108,7 +108,8 @@ def __init__(self, address, username=None, password=None, debug=False): """ self.container_id = "eventhub.pysdk-" + str(uuid.uuid4())[:8] self.address = urlparse(address) - self.mgmt_node = b"$management" + self.eh_name = self.address.path.lstrip('/') + self.mgmt_target = "amqps://{}/{}".format(self.address.hostname, self.eh_name) url_username = unquote_plus(self.address.username) if self.address.username else None username = username or url_username url_password = unquote_plus(self.address.password) if self.address.password else None @@ -117,8 +118,7 @@ def __init__(self, address, username=None, password=None, debug=False): raise ValueError("Missing username and/or password.") self.auth_uri = "sb://{}{}".format(self.address.hostname, self.address.path) self._auth_config = {'username': username, 'password': password} - self.auth = self._create_auth() - self.connection = None + self.get_auth = functools.partial(self._create_auth) self.debug = debug self.clients = [] @@ -148,10 +148,10 @@ def from_iothub_connection_string(cls, conn_str, **kwargs): password = _generate_sas_token(address, policy, key) client = cls("amqps://" + address, username=username, password=password, **kwargs) client._auth_config = { - 'username': policy, - 'password': key, - 'iot_username': username, - 'iot_password': password} # pylint: disable=protected-access + 'iot_username': policy, + 'iot_password': key, + 'username': username, + 'password': password} # pylint: disable=protected-access return client def _create_auth(self, username=None, password=None): @@ -172,7 +172,7 @@ def _create_auth(self, username=None, password=None): return authentication.SASLPlain(self.address.hostname, username, password) return authentication.SASTokenAuth.from_shared_access_key(self.auth_uri, username, password, timeout=60) - def _create_properties(self): # pylint: disable=no-self-use + def create_properties(self): # pylint: disable=no-self-use """ Format the properties with which to instantiate the connection. This acts like a user agent over HTTP. @@ -186,29 +186,6 @@ def _create_properties(self): # pylint: disable=no-self-use properties["platform"] = sys.platform return properties - def _create_connection(self): - """ - Create a new ~uamqp.connection.Connection instance that will be shared between all - Sender/Receiver clients. - """ - if not self.connection: - log.info("{}: Creating connection with address={}".format( - self.container_id, self.address.geturl())) - self.connection = Connection( - self.address.hostname, - self.auth, - container_id=self.container_id, - properties=self._create_properties(), - debug=self.debug) - - def _close_connection(self): - """ - Close and destroy the connection. - """ - if self.connection: - self.connection.destroy() - self.connection = None - def _close_clients(self): """ Close all open Sender/Receiver clients. @@ -219,25 +196,26 @@ def _close_clients(self): def _start_clients(self): for client in self.clients: try: - client.open(self.connection) - while not client.has_started(): - self.connection.work() + client.open() except Exception as exp: # pylint: disable=broad-except client.close(exception=exp) + def _process_redirect_uri(self, redirect): + redirect_uri = redirect.address.decode('utf-8') + auth_uri, _, _ = redirect_uri.partition("/ConsumerGroups") + self.address = urlparse(auth_uri) + self.auth_uri = "sb://{}{}".format(self.address.hostname, self.address.path) + self.eh_name = self.address.path.lstrip('/') + self.mgmt_target = redirect_uri + def _handle_redirect(self, redirects): if len(redirects) != len(self.clients): raise EventHubError("Some clients are attempting to redirect the connection.") if not all(r.hostname == redirects[0].hostname for r in redirects): raise EventHubError("Multiple clients attempting to redirect to different hosts.") - self.auth_uri = redirects[0].address.decode('utf-8') - self.auth = self._create_auth() - new_target, _, _ = self.auth_uri.partition("/ConsumerGroups") - self.address = urlparse(new_target) - self.mgmt_node = new_target.encode('UTF-8') + b"/$management" - self.connection.redirect(redirects[0], self.auth) + self._process_redirect_uri(redirects[0]) for client in self.clients: - client.open(self.connection) + client.open() def run(self): """ @@ -252,7 +230,6 @@ def run(self): :rtype: list[~azure.eventhub.common.EventHubError] """ log.info("{}: Starting {} clients".format(self.container_id, len(self.clients))) - self._create_connection() try: self._start_clients() redirects = [c.redirected for c in self.clients if c.redirected] @@ -279,7 +256,6 @@ def stop(self): log.info("{}: Stopping {} clients".format(self.container_id, len(self.clients))) self.stopped = True self._close_clients() - self._close_connection() def get_eventhub_info(self): """ @@ -293,13 +269,14 @@ def get_eventhub_info(self): :rtype: dict """ - eh_name = self.address.path.lstrip('/') - target = "amqps://{}/{}".format(self.address.hostname, eh_name) - mgmt_auth = self._create_auth() - mgmt_client = uamqp.AMQPClient(target, auth=mgmt_auth, debug=self.debug) + alt_creds = { + "username": self._auth_config.get("iot_username"), + "password":self._auth_config.get("iot_password")} try: + mgmt_auth = self._create_auth(**alt_creds) + mgmt_client = uamqp.AMQPClient(self.mgmt_target, auth=mgmt_auth, debug=self.debug) mgmt_client.open() - mgmt_msg = Message(application_properties={'name': eh_name}) + mgmt_msg = Message(application_properties={'name': self.eh_name}) response = mgmt_client.mgmt_request( mgmt_msg, constants.READ_OPERATION, @@ -340,10 +317,7 @@ def add_receiver(self, consumer_group, partition, offset=None, prefetch=300, ope path = self.address.path + operation if operation else self.address.path source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( self.address.hostname, path, consumer_group, partition) - source = Source(source_url) - if offset is not None: - source.set_filter(offset.selector()) - handler = Receiver(self, source, prefetch=prefetch) + handler = Receiver(self, source_url, offset=offset, prefetch=prefetch) self.clients.append(handler) return handler diff --git a/azure/eventhub/common.py b/azure/eventhub/common.py index f14e778..a89eebc 100644 --- a/azure/eventhub/common.py +++ b/azure/eventhub/common.py @@ -10,6 +10,13 @@ from uamqp import types, constants, errors from uamqp.message import MessageHeader, MessageProperties +_NO_RETRY_ERRORS = ( + b"com.microsoft:argument-out-of-range", + b"com.microsoft:entity-disabled", + b"com.microsoft:auth-failed", + b"com.microsoft:precondition-failed", + b"com.microsoft:argument-error" +) def _error_handler(error): """ @@ -29,7 +36,9 @@ def _error_handler(error): elif error.condition == b'com.microsoft:operation-cancelled': return errors.ErrorAction(retry=True) elif error.condition == b"com.microsoft:container-close": - return errors.ErrorAction(retry=True) + return errors.ErrorAction(retry=True, backoff=4) + elif error.condition in _NO_RETRY_ERRORS: + return errors.ErrorAction(retry=False) return errors.ErrorAction(retry=True) @@ -97,7 +106,7 @@ def offset(self): :rtype: int """ try: - return self._annotations[EventData.PROP_OFFSET].decode('UTF-8') + return Offset(self._annotations[EventData.PROP_OFFSET].decode('UTF-8')) except (KeyError, AttributeError): return None diff --git a/azure/eventhub/receiver.py b/azure/eventhub/receiver.py index a21b30f..7e0c5e6 100644 --- a/azure/eventhub/receiver.py +++ b/azure/eventhub/receiver.py @@ -4,9 +4,9 @@ # -------------------------------------------------------------------------------------------- from uamqp import types, errors -from uamqp import ReceiveClient +from uamqp import ReceiveClient, Source -from azure.eventhub.common import EventHubError, EventData, Offset +from azure.eventhub.common import EventHubError, EventData, Offset, _error_handler class Receiver: @@ -16,38 +16,46 @@ class Receiver: timeout = 0 _epoch = b'com.microsoft:epoch' - def __init__(self, client, source, prefetch=300, epoch=None): + def __init__(self, client, source, offset=None, prefetch=300, epoch=None): """ Instantiate a receiver. :param client: The parent EventHubClient. :type client: ~azure.eventhub.client.EventHubClient :param source: The source EventHub from which to receive events. - :type source: ~uamqp.address.Source + :type source: str :param prefetch: The number of events to prefetch from the service for processing. Default is 300. :type prefetch: int :param epoch: An optional epoch value. :type epoch: int """ - self.offset = None + self.client = client + self.source = source + self.offset = offset self.prefetch = prefetch self.epoch = epoch + self.retry_policy = errors.ErrorPolicy(max_retries=3, on_error=_error_handler) self.properties = None self.redirected = None - self.debug = client.debug self.error = None + source = Source(self.source) + if self.offset is not None: + source.set_filter(self.offset.selector()) if epoch: self.properties = {types.AMQPSymbol(self._epoch): types.AMQPLong(int(epoch))} self._handler = ReceiveClient( source, - auth=client.auth, - debug=self.debug, + auth=self.client.get_auth(), + debug=self.client.debug, prefetch=self.prefetch, link_properties=self.properties, - timeout=self.timeout) + timeout=self.timeout, + error_policy=self.retry_policy, + keep_alive_interval=30, + properties=self.client.create_properties()) - def open(self, connection): + def open(self): """ Open the Receiver using the supplied conneciton. If the handler has previously been redirected, the redirect @@ -57,14 +65,50 @@ def open(self, connection): :type: connection: ~uamqp.connection.Connection """ if self.redirected: + self.source = self.redirected.address + source = Source(self.source) + if self.offset is not None: + source.set_filter(self.offset.selector()) + alt_creds = { + "username": self.client._auth_config.get("iot_username"), + "password":self.client._auth_config.get("iot_password")} self._handler = ReceiveClient( - self.redirected.address, - auth=None, - debug=self.debug, + source, + auth=self.client.get_auth(**alt_creds), + debug=self.client.debug, prefetch=self.prefetch, link_properties=self.properties, - timeout=self.timeout) - self._handler.open(connection) + timeout=self.timeout, + error_policy=self.retry_policy, + keep_alive_interval=30, + properties=self.client.create_properties()) + self._handler.open() + while not self.has_started(): + self._handler._connection.work() + + def reconnect(self): + """If the Receiver was disconnected from the service with + a retryable error - attempt to reconnect.""" + alt_creds = { + "username": self.client._auth_config.get("iot_username"), + "password":self.client._auth_config.get("iot_password")} + self._handler.close() + source = Source(self.source) + if self.offset is not None: + source.set_filter(self.offset.selector()) + self._handler = ReceiveClient( + source, + auth=self.client.get_auth(**alt_creds), + debug=self.client.debug, + prefetch=self.prefetch, + link_properties=self.properties, + timeout=self.timeout, + error_policy=self.retry_policy, + keep_alive_interval=30, + properties=self.client.create_properties()) + self._handler.open() + while not self.has_started(): + self._handler._connection.work() def get_handler_state(self): """ @@ -146,34 +190,29 @@ def receive(self, max_batch_size=None, timeout=None): """ if self.error: raise self.error + data_batch = [] try: timeout_ms = 1000 * timeout if timeout else 0 message_batch = self._handler.receive_message_batch( max_batch_size=max_batch_size, timeout=timeout_ms) - data_batch = [] for message in message_batch: event_data = EventData(message=message) self.offset = event_data.offset data_batch.append(event_data) return data_batch - except errors.LinkDetach as detach: - error = EventHubError(str(detach), detach) - self.close(exception=error) - raise error + except (errors.LinkDetach, errors.ConnectionClose) as shutdown: + if shutdown.action.retry: + self.reconnect() + return data_batch + else: + error = EventHubError(str(shutdown), shutdown) + self.close(exception=error) + raise error + except (errors.MessageHandlerError): + self.reconnect() + return data_batch except Exception as e: error = EventHubError("Receive failed: {}".format(e)) self.close(exception=error) raise error - - def selector(self, default): - """ - Create a selector for the current offset if it is set. - - :param default: The fallback receive offset. - :type default: ~azure.eventhub.common.Offset - :rtype: ~azure.eventhub.common.Offset - """ - if self.offset is not None: - return Offset(self.offset).selector() - return default diff --git a/azure/eventhub/sender.py b/azure/eventhub/sender.py index f635da8..ccc69d8 100644 --- a/azure/eventhub/sender.py +++ b/azure/eventhub/sender.py @@ -27,24 +27,26 @@ def __init__(self, client, target, partition=None): :param target: The URI of the EventHub to send to. :type target: str """ - self.conneciton = None + self.client = client + self.target = target + self.partition = partition self.redirected = None self.error = None - self.debug = client.debug - self.partition = partition self.retry_policy = errors.ErrorPolicy(max_retries=3, on_error=_error_handler) if partition: - target += "/Partitions/" + partition + self.target += "/Partitions/" + partition self._handler = SendClient( - target, - auth=client.auth, - debug=self.debug, + self.target, + auth=self.client.get_auth(), + debug=self.client.debug, msg_timeout=Sender.TIMEOUT, - error_policy=self.retry_policy) + error_policy=self.retry_policy, + keep_alive_interval=30, + properties=self.client.create_properties()) self._outcome = None self._condition = None - def open(self, connection): + def open(self): #, connection): """ Open the Sender using the supplied conneciton. If the handler has previously been redirected, the redirect @@ -53,29 +55,41 @@ def open(self, connection): :param connection: The underlying client shared connection. :type: connection: ~uamqp.connection.Connection """ - self.connection = connection if self.redirected: + self.target = self.redirected.address + alt_creds = { + "username": self.client._auth_config.get("iot_username"), + "password":self.client._auth_config.get("iot_password")} self._handler = SendClient( - self.redirected.address, - auth=None, - debug=self.debug, + self.target, + auth=self.client.get_auth(**alt_creds), + debug=self.client.debug, msg_timeout=Sender.TIMEOUT, - retry_policy=self.retry_policy) - self._handler.open(connection) + error_policy=self.retry_policy, + keep_alive_interval=30, + properties=self.client.create_properties()) + self._handler.open() #connection) + while not self.has_started(): + self._handler._connection.work() def reconnect(self): """If the Sender was disconnected from the service with a retryable error - attempt to reconnect.""" pending_states = (constants.MessageState.WaitingForSendAck, constants.MessageState.WaitingToBeSent) unsent_events = [e for e in self._handler._pending_messages if e.state in pending_states] + alt_creds = { + "username": self.client._auth_config.get("iot_username"), + "password":self.client._auth_config.get("iot_password")} self._handler.close() self._handler = SendClient( - self.redirected.address, - auth=None, - debug=self.debug, + self.target, + auth=self.client.get_auth(**alt_creds), + debug=self.client.debug, msg_timeout=Sender.TIMEOUT, - retry_policy=self.retry_policy) - self._handler.open(self.connection) + error_policy=self.retry_policy, + keep_alive_interval=30, + properties=self.client.create_properties()) + self._handler.open() self._handler._pending_messages = unsent_events self._handler.wait() @@ -158,17 +172,15 @@ def send(self, event_data): error = EventHubError(str(failed), failed) self.close(exception=error) raise error - except errors.LinkDetach as detach: - if detach.action.retry: + except (errors.LinkDetach, errors.ConnectionClose) as shutdown: + if shutdown.action.retry: self.reconnect() else: - error = EventHubError(str(detach), detach) + error = EventHubError(str(shutdown), shutdown) self.close(exception=error) raise error - except errors.ConnectionClose as close: - error = EventHubError(str(close), close) - self.close(exception=error) - raise error + except (errors.MessageHandlerError): + self.reconnect() except Exception as e: error = EventHubError("Send failed: {}".format(e)) self.close(exception=error) @@ -202,17 +214,15 @@ def wait(self): raise self.error try: self._handler.wait() - except errors.LinkDetach as detach: - if detach.action.retry: + except (errors.LinkDetach, errors.ConnectionClose) as shutdown: + if shutdown.action.retry: self.reconnect() else: - error = EventHubError(str(detach), detach) + error = EventHubError(str(shutdown), shutdown) self.close(exception=error) raise error - except errors.ConnectionClose as close: - error = EventHubError(str(close), close) - self.close(exception=error) - raise error + except (errors.MessageHandlerError): + self.reconnect() except Exception as e: raise EventHubError("Send failed: {}".format(e)) From 35e1a67aaec4feda5a9503cb441e1a0a6c0b261b Mon Sep 17 00:00:00 2001 From: annatisch Date: Thu, 26 Jul 2018 12:29:50 -0700 Subject: [PATCH 20/52] Added HTTP proxy support Fix for issue #41 --- azure/eventhub/_async/__init__.py | 6 ++++-- azure/eventhub/client.py | 13 ++++++++++--- azure/eventprocessorhost/eh_partition_pump.py | 3 ++- azure/eventprocessorhost/eph.py | 1 + azure/eventprocessorhost/partition_manager.py | 3 ++- 5 files changed, 19 insertions(+), 7 deletions(-) diff --git a/azure/eventhub/_async/__init__.py b/azure/eventhub/_async/__init__.py index 3041da5..16c6b77 100644 --- a/azure/eventhub/_async/__init__.py +++ b/azure/eventhub/_async/__init__.py @@ -55,8 +55,10 @@ def _create_auth(self, username=None, password=None): # pylint: disable=no-self username = username or self._auth_config['username'] password = password or self._auth_config['password'] if "@sas.root" in username: - return authentication.SASLPlain(self.address.hostname, username, password) - return authentication.SASTokenAsync.from_shared_access_key(self.auth_uri, username, password, timeout=60) + return authentication.SASLPlain( + self.address.hostname, username, password, http_proxy=self.http_proxy) + return authentication.SASTokenAsync.from_shared_access_key( + self.auth_uri, username, password, timeout=60, http_proxy=self.http_proxy) async def _close_clients_async(self): """ diff --git a/azure/eventhub/client.py b/azure/eventhub/client.py index 1659f70..6ac4dde 100644 --- a/azure/eventhub/client.py +++ b/azure/eventhub/client.py @@ -89,7 +89,7 @@ class EventHubClient(object): events to and receiving events from the Azure Event Hubs service. """ - def __init__(self, address, username=None, password=None, debug=False): + def __init__(self, address, username=None, password=None, debug=False, http_proxy=None): """ Constructs a new EventHubClient with the given address URL. @@ -105,10 +105,15 @@ def __init__(self, address, username=None, password=None, debug=False): :param debug: Whether to output network trace logs to the logger. Default is `False`. :type debug: bool + :param http_proxy: HTTP proxy settings. This must be a dictionary with the following + keys: 'proxy_hostname' (str value) and 'proxy_port' (int value). + Additionally the following keys may also be present: 'username', 'password'. + :type http_proxy: dict[str, Any] """ self.container_id = "eventhub.pysdk-" + str(uuid.uuid4())[:8] self.address = urlparse(address) self.eh_name = self.address.path.lstrip('/') + self.http_proxy = http_proxy self.mgmt_target = "amqps://{}/{}".format(self.address.hostname, self.eh_name) url_username = unquote_plus(self.address.username) if self.address.username else None username = username or url_username @@ -169,8 +174,10 @@ def _create_auth(self, username=None, password=None): username = username or self._auth_config['username'] password = password or self._auth_config['password'] if "@sas.root" in username: - return authentication.SASLPlain(self.address.hostname, username, password) - return authentication.SASTokenAuth.from_shared_access_key(self.auth_uri, username, password, timeout=60) + return authentication.SASLPlain( + self.address.hostname, username, password, http_proxy=self.http_proxy) + return authentication.SASTokenAuth.from_shared_access_key( + self.auth_uri, username, password, timeout=60, http_proxy=self.http_proxy) def create_properties(self): # pylint: disable=no-self-use """ diff --git a/azure/eventprocessorhost/eh_partition_pump.py b/azure/eventprocessorhost/eh_partition_pump.py index 86c42d2..4801a25 100644 --- a/azure/eventprocessorhost/eh_partition_pump.py +++ b/azure/eventprocessorhost/eh_partition_pump.py @@ -66,7 +66,8 @@ async def open_clients_async(self): # Create event hub client and receive handler and set options self.eh_client = EventHubClientAsync( self.host.eh_config.client_address, - debug=self.host.eph_options.debug_trace) + debug=self.host.eph_options.debug_trace, + http_proxy=self.host.eph_options.http_proxy) self.partition_receive_handler = self.eh_client.add_async_receiver( self.partition_context.consumer_group_name, self.partition_context.partition_id, diff --git a/azure/eventprocessorhost/eph.py b/azure/eventprocessorhost/eph.py index 27cbc3e..7c7541e 100644 --- a/azure/eventprocessorhost/eph.py +++ b/azure/eventprocessorhost/eph.py @@ -73,3 +73,4 @@ def __init__(self): self.release_pump_on_timeout = False self.initial_offset_provider = "-1" self.debug_trace = False + self.http_proxy = None diff --git a/azure/eventprocessorhost/partition_manager.py b/azure/eventprocessorhost/partition_manager.py index da58423..7025fe0 100644 --- a/azure/eventprocessorhost/partition_manager.py +++ b/azure/eventprocessorhost/partition_manager.py @@ -38,7 +38,8 @@ async def get_partition_ids_async(self): try: eh_client = EventHubClientAsync( self.host.eh_config.client_address, - debug=self.host.eph_options.debug_trace) + debug=self.host.eph_options.debug_trace, + http_proxy=self.host.eph_options.http_proxy) try: eh_info = await eh_client.get_eventhub_info_async() self.partition_ids = eh_info['partition_ids'] From 0f5ddda3637706baa75f2f385259e1f50b142809 Mon Sep 17 00:00:00 2001 From: annatisch Date: Thu, 26 Jul 2018 13:43:02 -0700 Subject: [PATCH 21/52] Fixed some tests + samples --- azure/eventhub/_async/sender_async.py | 10 ++-------- azure/eventhub/sender.py | 10 ++-------- azure/eventprocessorhost/partition_context.py | 4 ++-- examples/recv.py | 2 +- examples/recv_async.py | 2 +- examples/recv_batch.py | 2 +- tests/test_iothub_receive.py | 2 ++ tests/test_iothub_send.py | 1 - tests/test_receive.py | 4 ++++ 9 files changed, 15 insertions(+), 22 deletions(-) diff --git a/azure/eventhub/_async/sender_async.py b/azure/eventhub/_async/sender_async.py index 3a10d6f..cf7174e 100644 --- a/azure/eventhub/_async/sender_async.py +++ b/azure/eventhub/_async/sender_async.py @@ -59,12 +59,9 @@ async def open_async(self): """ if self.redirected: self.target = self.redirected.address - alt_creds = { - "username": self.client._auth_config.get("iot_username"), - "password":self.client._auth_config.get("iot_password")} self._handler = SendClientAsync( self.target, - auth=self.client.get_auth(**alt_creds), + auth=self.client.get_auth(), debug=self.client.debug, msg_timeout=Sender.TIMEOUT, error_policy=self.retry_policy, @@ -80,13 +77,10 @@ async def reconnect_async(self): a retryable error - attempt to reconnect.""" pending_states = (constants.MessageState.WaitingForSendAck, constants.MessageState.WaitingToBeSent) unsent_events = [e for e in self._handler._pending_messages if e.state in pending_states] - alt_creds = { - "username": self.client._auth_config.get("iot_username"), - "password":self.client._auth_config.get("iot_password")} await self._handler.close_async() self._handler = SendClientAsync( self.target, - auth=self.client.get_auth(**alt_creds), + auth=self.client.get_auth(), debug=self.client.debug, msg_timeout=Sender.TIMEOUT, error_policy=self.retry_policy, diff --git a/azure/eventhub/sender.py b/azure/eventhub/sender.py index ccc69d8..e4455c4 100644 --- a/azure/eventhub/sender.py +++ b/azure/eventhub/sender.py @@ -57,12 +57,9 @@ def open(self): #, connection): """ if self.redirected: self.target = self.redirected.address - alt_creds = { - "username": self.client._auth_config.get("iot_username"), - "password":self.client._auth_config.get("iot_password")} self._handler = SendClient( self.target, - auth=self.client.get_auth(**alt_creds), + auth=self.client.get_auth(), debug=self.client.debug, msg_timeout=Sender.TIMEOUT, error_policy=self.retry_policy, @@ -77,13 +74,10 @@ def reconnect(self): a retryable error - attempt to reconnect.""" pending_states = (constants.MessageState.WaitingForSendAck, constants.MessageState.WaitingToBeSent) unsent_events = [e for e in self._handler._pending_messages if e.state in pending_states] - alt_creds = { - "username": self.client._auth_config.get("iot_username"), - "password":self.client._auth_config.get("iot_password")} self._handler.close() self._handler = SendClient( self.target, - auth=self.client.get_auth(**alt_creds), + auth=self.client.get_auth(), debug=self.client.debug, msg_timeout=Sender.TIMEOUT, error_policy=self.retry_policy, diff --git a/azure/eventprocessorhost/partition_context.py b/azure/eventprocessorhost/partition_context.py index 9eaf53f..b21514b 100644 --- a/azure/eventprocessorhost/partition_context.py +++ b/azure/eventprocessorhost/partition_context.py @@ -34,7 +34,7 @@ def set_offset_and_sequence_number(self, event_data): """ if not event_data: raise Exception(event_data) - self.offset = event_data.offset + self.offset = event_data.offset.value self.sequence_number = event_data.sequence_number async def get_initial_offset_async(self): # throws InterruptedException, ExecutionException @@ -84,7 +84,7 @@ async def checkpoint_async_event_data(self, event_data): raise ValueError("Argument Out Of Range event_data x-opt-sequence-number") await self.persist_checkpoint_async(Checkpoint(self.partition_id, - event_data.offset, + event_data.offset.value, event_data.sequence_number)) def to_string(self): diff --git a/examples/recv.py b/examples/recv.py index 92a5df2..d2fbdf7 100644 --- a/examples/recv.py +++ b/examples/recv.py @@ -41,7 +41,7 @@ for event_data in receiver.receive(timeout=100): last_offset = event_data.offset last_sn = event_data.sequence_number - print("Received: {}, {}".format(last_offset, last_sn)) + print("Received: {}, {}".format(last_offset.value, last_sn)) total += 1 end_time = time.time() diff --git a/examples/recv_async.py b/examples/recv_async.py index ab8da39..04d9226 100644 --- a/examples/recv_async.py +++ b/examples/recv_async.py @@ -39,7 +39,7 @@ async def pump(client, partition): for event_data in await receiver.receive(timeout=10): last_offset = event_data.offset last_sn = event_data.sequence_number - print("Received: {}, {}".format(last_offset, last_sn)) + print("Received: {}, {}".format(last_offset.value, last_sn)) total += 1 end_time = time.time() run_time = end_time - start_time diff --git a/examples/recv_batch.py b/examples/recv_batch.py index 7ce562d..9478f51 100644 --- a/examples/recv_batch.py +++ b/examples/recv_batch.py @@ -40,7 +40,7 @@ client.run() batched_events = receiver.receive(max_batch_size=10) for event_data in batched_events: - last_offset = event_data.offset + last_offset = event_data.offset.value last_sn = event_data.sequence_number total += 1 print("Partition {}, Received {}, sn={} offset={}".format( diff --git a/tests/test_iothub_receive.py b/tests/test_iothub_receive.py index a48274b..ced6858 100644 --- a/tests/test_iothub_receive.py +++ b/tests/test_iothub_receive.py @@ -16,6 +16,8 @@ def test_iothub_receive_sync(iot_connection_str, device_id): receiver = client.add_receiver("$default", "0", operation='/messages/events') try: client.run() + partitions = client.get_eventhub_info() + assert partitions["partition_ids"] == ["0", "1", "2", "3"] received = receiver.receive(timeout=5) assert len(received) == 0 finally: diff --git a/tests/test_iothub_send.py b/tests/test_iothub_send.py index 3f39c61..7c0dd7c 100644 --- a/tests/test_iothub_send.py +++ b/tests/test_iothub_send.py @@ -20,7 +20,6 @@ def test_iothub_send_single_event(iot_connection_str, device_id): sender = client.add_sender(operation='/messages/devicebound') try: client.run() - partitions = client.get_eventhub_info() outcome = sender.send(EventData(b"A single event", to_device=device_id)) assert outcome.value == 0 except: diff --git a/tests/test_receive.py b/tests/test_receive.py index fda5a96..1b7480e 100644 --- a/tests/test_receive.py +++ b/tests/test_receive.py @@ -33,9 +33,13 @@ def test_receive_end_of_stream(connection_str, senders): def test_receive_with_offset_sync(connection_str, senders): client = EventHubClient.from_connection_string(connection_str, debug=False) + partitions = client.get_eventhub_info() + assert partitions["partition_ids"] == ["0", "1"] receiver = client.add_receiver("$default", "0", offset=Offset('@latest')) try: client.run() + more_partitions = client.get_eventhub_info() + assert more_partitions["partition_ids"] == ["0", "1"] received = receiver.receive(timeout=5) assert len(received) == 0 From 5130b2a39a5a26f46e7dc245d770da0d0c587b71 Mon Sep 17 00:00:00 2001 From: annatisch Date: Thu, 26 Jul 2018 13:54:41 -0700 Subject: [PATCH 22/52] pylint fixes --- azure/eventhub/_async/__init__.py | 4 +--- azure/eventhub/_async/receiver_async.py | 4 +++- azure/eventhub/_async/sender_async.py | 9 +++++---- azure/eventhub/client.py | 5 ++--- azure/eventhub/common.py | 4 ++-- azure/eventhub/receiver.py | 6 ++++-- azure/eventhub/sender.py | 12 +++++------- 7 files changed, 22 insertions(+), 22 deletions(-) diff --git a/azure/eventhub/_async/__init__.py b/azure/eventhub/_async/__init__.py index 16c6b77..93a60c5 100644 --- a/azure/eventhub/_async/__init__.py +++ b/azure/eventhub/_async/__init__.py @@ -69,7 +69,7 @@ async def _close_clients_async(self): async def _wait_for_client(self, client): try: while client.get_handler_state().value == 2: - await client._handler._connection.work_async() + await client._handler._connection.work_async() # pylint: disable=protected-access except Exception as exp: # pylint: disable=broad-except await client.close_async(exception=exp) @@ -182,10 +182,8 @@ def add_async_receiver(self, consumer_group, partition, offset=None, prefetch=30 :rtype: ~azure.eventhub._async.receiver_async.ReceiverAsync """ path = self.address.path + operation if operation else self.address.path - source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( self.address.hostname, path, consumer_group, partition) - print("RECEIVER_PATH", source_url) handler = AsyncReceiver(self, source_url, offset=offset, prefetch=prefetch, loop=loop) self.clients.append(handler) return handler diff --git a/azure/eventhub/_async/receiver_async.py b/azure/eventhub/_async/receiver_async.py index 072e04a..6ba2c3a 100644 --- a/azure/eventhub/_async/receiver_async.py +++ b/azure/eventhub/_async/receiver_async.py @@ -68,6 +68,7 @@ async def open_async(self): :param connection: The underlying client shared connection. :type: connection: ~uamqp._async.connection_async.ConnectionAsync """ + # pylint: disable=protected-access if self.redirected: self.source = self.redirected.address source = Source(self.source) @@ -93,6 +94,7 @@ async def open_async(self): async def reconnect_async(self): """If the Receiver was disconnected from the service with a retryable error - attempt to reconnect.""" + # pylint: disable=protected-access alt_creds = { "username": self.client._auth_config.get("iot_username"), "password":self.client._auth_config.get("iot_password")} @@ -194,7 +196,7 @@ async def receive(self, max_batch_size=None, timeout=None): error = EventHubError(str(shutdown), shutdown) await self.close_async(exception=error) raise error - except (errors.MessageHandlerError): + except errors.MessageHandlerError: await self.reconnect_async() return data_batch except Exception as e: diff --git a/azure/eventhub/_async/sender_async.py b/azure/eventhub/_async/sender_async.py index cf7174e..42865f3 100644 --- a/azure/eventhub/_async/sender_async.py +++ b/azure/eventhub/_async/sender_async.py @@ -70,11 +70,12 @@ async def open_async(self): loop=self.loop) await self._handler.open_async() while not await self.has_started(): - await self._handler._connection.work_async() + await self._handler._connection.work_async() # pylint: disable=protected-access async def reconnect_async(self): """If the Receiver was disconnected from the service with a retryable error - attempt to reconnect.""" + # pylint: disable=protected-access pending_states = (constants.MessageState.WaitingForSendAck, constants.MessageState.WaitingToBeSent) unsent_events = [e for e in self._handler._pending_messages if e.state in pending_states] await self._handler.close_async() @@ -130,7 +131,7 @@ async def close_async(self, exception=None): elif isinstance(exception, EventHubError): self.error = exception elif isinstance(exception, (errors.LinkDetach, errors.ConnectionClose)): - self.error = EventHubError(str(error), error) + self.error = EventHubError(str(exception), exception) elif exception: self.error = EventHubError(str(exception)) else: @@ -163,7 +164,7 @@ async def send(self, event_data): error = EventHubError(str(shutdown), shutdown) await self.close_async(exception=error) raise error - except (errors.MessageHandlerError): + except errors.MessageHandlerError: await self.reconnect_async() except Exception as e: error = EventHubError("Send failed: {}".format(e)) @@ -187,7 +188,7 @@ async def wait_async(self): error = EventHubError(str(shutdown), shutdown) await self.close_async(exception=error) raise error - except (errors.MessageHandlerError): + except errors.MessageHandlerError: await self.reconnect_async() except Exception as e: raise EventHubError("Send failed: {}".format(e)) diff --git a/azure/eventhub/client.py b/azure/eventhub/client.py index 6ac4dde..9c57cbd 100644 --- a/azure/eventhub/client.py +++ b/azure/eventhub/client.py @@ -15,7 +15,6 @@ from urllib.parse import urlparse, unquote_plus, urlencode, quote_plus import uamqp -from uamqp import Connection from uamqp import Message from uamqp import authentication from uamqp import constants @@ -152,11 +151,11 @@ def from_iothub_connection_string(cls, conn_str, **kwargs): username = "{}@sas.root.{}".format(policy, hub_name) password = _generate_sas_token(address, policy, key) client = cls("amqps://" + address, username=username, password=password, **kwargs) - client._auth_config = { + client._auth_config = { # pylint: disable=protected-access 'iot_username': policy, 'iot_password': key, 'username': username, - 'password': password} # pylint: disable=protected-access + 'password': password} return client def _create_auth(self, username=None, password=None): diff --git a/azure/eventhub/common.py b/azure/eventhub/common.py index a89eebc..035a812 100644 --- a/azure/eventhub/common.py +++ b/azure/eventhub/common.py @@ -257,7 +257,7 @@ def __init__(self, message, details=None): self._parse_error(details.description) for detail in self.details: self.message += "\n{}".format(detail) - except: + except: # pylint: disable=bare-except self.message += "\n{}".format(details) super(EventHubError, self).__init__(self.message) @@ -268,7 +268,7 @@ def _parse_error(self, error_list): if details_index >= 0: details_msg = self.message[details_index + 1:] self.message = self.message[0:details_index] - + tracking_index = details_msg.index(", TrackingId:") system_index = details_msg.index(", SystemTracker:") timestamp_index = details_msg.index(", Timestamp:") diff --git a/azure/eventhub/receiver.py b/azure/eventhub/receiver.py index 7e0c5e6..49a15ce 100644 --- a/azure/eventhub/receiver.py +++ b/azure/eventhub/receiver.py @@ -6,7 +6,7 @@ from uamqp import types, errors from uamqp import ReceiveClient, Source -from azure.eventhub.common import EventHubError, EventData, Offset, _error_handler +from azure.eventhub.common import EventHubError, EventData, _error_handler class Receiver: @@ -64,6 +64,7 @@ def open(self): :param connection: The underlying client shared connection. :type: connection: ~uamqp.connection.Connection """ + # pylint: disable=protected-access if self.redirected: self.source = self.redirected.address source = Source(self.source) @@ -89,6 +90,7 @@ def open(self): def reconnect(self): """If the Receiver was disconnected from the service with a retryable error - attempt to reconnect.""" + # pylint: disable=protected-access alt_creds = { "username": self.client._auth_config.get("iot_username"), "password":self.client._auth_config.get("iot_password")} @@ -209,7 +211,7 @@ def receive(self, max_batch_size=None, timeout=None): error = EventHubError(str(shutdown), shutdown) self.close(exception=error) raise error - except (errors.MessageHandlerError): + except errors.MessageHandlerError: self.reconnect() return data_batch except Exception as e: diff --git a/azure/eventhub/sender.py b/azure/eventhub/sender.py index e4455c4..37d6024 100644 --- a/azure/eventhub/sender.py +++ b/azure/eventhub/sender.py @@ -3,9 +3,6 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -import time - -import uamqp from uamqp import constants, errors from uamqp import SendClient @@ -65,13 +62,14 @@ def open(self): #, connection): error_policy=self.retry_policy, keep_alive_interval=30, properties=self.client.create_properties()) - self._handler.open() #connection) + self._handler.open() while not self.has_started(): - self._handler._connection.work() + self._handler._connection.work() # pylint: disable=protected-access def reconnect(self): """If the Sender was disconnected from the service with a retryable error - attempt to reconnect.""" + # pylint: disable=protected-access pending_states = (constants.MessageState.WaitingForSendAck, constants.MessageState.WaitingToBeSent) unsent_events = [e for e in self._handler._pending_messages if e.state in pending_states] self._handler.close() @@ -173,7 +171,7 @@ def send(self, event_data): error = EventHubError(str(shutdown), shutdown) self.close(exception=error) raise error - except (errors.MessageHandlerError): + except errors.MessageHandlerError: self.reconnect() except Exception as e: error = EventHubError("Send failed: {}".format(e)) @@ -215,7 +213,7 @@ def wait(self): error = EventHubError(str(shutdown), shutdown) self.close(exception=error) raise error - except (errors.MessageHandlerError): + except errors.MessageHandlerError: self.reconnect() except Exception as e: raise EventHubError("Send failed: {}".format(e)) From f02c954a9cf731e451399af0955fc62660734d2f Mon Sep 17 00:00:00 2001 From: annatisch Date: Thu, 26 Jul 2018 14:56:16 -0700 Subject: [PATCH 23/52] bumped version --- HISTORY.rst | 13 +++++++++++++ azure/eventhub/__init__.py | 2 +- azure/eventhub/sender.py | 2 +- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index c90d34f..89db3a3 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,19 @@ Release History =============== +0.2.0rc2 (2018-07-29) ++++++++++++++++++++++ + +- **Breaking change** `EventData.offset` will now return an object of type `~uamqp.common.Offset` rather than str. + The original string value can be retrieved from `~uamqp.common.Offset.value`. +- Each sender/receiver will now run in its own independent connection. +- Updated uAMQP dependency to 0.2.0 +- Fixed issue with IoTHub clients not being able to retrieve partition information. +- Added support for HTTP proxy settings to both EventHubClient and EPH. +- Added error handling policy to automatically reconnect on retryable error. +- Added keep-alive thread for maintaining an unused connection. + + 0.2.0rc1 (2018-07-06) +++++++++++++++++++++ diff --git a/azure/eventhub/__init__.py b/azure/eventhub/__init__.py index 5182b38..5acadea 100644 --- a/azure/eventhub/__init__.py +++ b/azure/eventhub/__init__.py @@ -3,7 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -__version__ = "0.2.0rc1" +__version__ = "0.2.0rc2" from azure.eventhub.common import EventData, EventHubError, Offset from azure.eventhub.client import EventHubClient diff --git a/azure/eventhub/sender.py b/azure/eventhub/sender.py index 37d6024..ff358d0 100644 --- a/azure/eventhub/sender.py +++ b/azure/eventhub/sender.py @@ -43,7 +43,7 @@ def __init__(self, client, target, partition=None): self._outcome = None self._condition = None - def open(self): #, connection): + def open(self): """ Open the Sender using the supplied conneciton. If the handler has previously been redirected, the redirect From 91c5d221d1e5fe76c2c619b3d533d6d991ae0ad9 Mon Sep 17 00:00:00 2001 From: annatisch Date: Wed, 1 Aug 2018 13:59:19 -0700 Subject: [PATCH 24/52] Added keepalive config and some eph fixes --- .gitignore | 1 + azure/eventhub/_async/__init__.py | 12 ++++++------ azure/eventhub/_async/receiver_async.py | 9 +++++---- azure/eventhub/_async/sender_async.py | 9 +++++---- azure/eventhub/client.py | 12 ++++++------ azure/eventhub/receiver.py | 9 +++++---- azure/eventhub/sender.py | 9 +++++---- .../azure_storage_checkpoint_manager.py | 6 +++--- azure/eventprocessorhost/eh_partition_pump.py | 8 +++++++- azure/eventprocessorhost/partition_context.py | 4 +++- azure/eventprocessorhost/partition_manager.py | 1 + 11 files changed, 47 insertions(+), 33 deletions(-) diff --git a/.gitignore b/.gitignore index ef97309..6b8bf2d 100644 --- a/.gitignore +++ b/.gitignore @@ -110,4 +110,5 @@ ENV/ azure/mgmt/ azure/common/ azure/profiles/ +azure/servicebus/ features/steps/mgmt_settings_real.py diff --git a/azure/eventhub/_async/__init__.py b/azure/eventhub/_async/__init__.py index 93a60c5..126fdc8 100644 --- a/azure/eventhub/_async/__init__.py +++ b/azure/eventhub/_async/__init__.py @@ -164,7 +164,7 @@ async def get_eventhub_info_async(self): finally: await mgmt_client.close_async() - def add_async_receiver(self, consumer_group, partition, offset=None, prefetch=300, operation=None, loop=None): + def add_async_receiver(self, consumer_group, partition, offset=None, prefetch=300, operation=None, keep_alive=30, loop=None): """ Add an async receiver to the client for a particular consumer group and partition. @@ -184,11 +184,11 @@ def add_async_receiver(self, consumer_group, partition, offset=None, prefetch=30 path = self.address.path + operation if operation else self.address.path source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( self.address.hostname, path, consumer_group, partition) - handler = AsyncReceiver(self, source_url, offset=offset, prefetch=prefetch, loop=loop) + handler = AsyncReceiver(self, source_url, offset=offset, prefetch=prefetch, keep_alive=keep_alive, loop=loop) self.clients.append(handler) return handler - def add_async_epoch_receiver(self, consumer_group, partition, epoch, prefetch=300, operation=None, loop=None): + def add_async_epoch_receiver(self, consumer_group, partition, epoch, prefetch=300, operation=None, keep_alive=30, loop=None): """ Add an async receiver to the client with an epoch value. Only a single epoch receiver can connect to a partition at any given time - additional epoch receivers must have @@ -211,11 +211,11 @@ def add_async_epoch_receiver(self, consumer_group, partition, epoch, prefetch=30 path = self.address.path + operation if operation else self.address.path source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( self.address.hostname, path, consumer_group, partition) - handler = AsyncReceiver(self, source_url, prefetch=prefetch, epoch=epoch, loop=loop) + handler = AsyncReceiver(self, source_url, prefetch=prefetch, epoch=epoch, keep_alive=keep_alive, loop=loop) self.clients.append(handler) return handler - def add_async_sender(self, partition=None, operation=None, loop=None): + def add_async_sender(self, partition=None, operation=None, keep_alive=30, loop=None): """ Add an async sender to the client to send ~azure.eventhub.common.EventData object to an EventHub. @@ -232,6 +232,6 @@ def add_async_sender(self, partition=None, operation=None, loop=None): target = "amqps://{}{}".format(self.address.hostname, self.address.path) if operation: target = target + operation - handler = AsyncSender(self, target, partition=partition, loop=loop) + handler = AsyncSender(self, target, partition=partition, keep_alive=keep_alive, loop=loop) self.clients.append(handler) return handler diff --git a/azure/eventhub/_async/receiver_async.py b/azure/eventhub/_async/receiver_async.py index 6ba2c3a..22166d9 100644 --- a/azure/eventhub/_async/receiver_async.py +++ b/azure/eventhub/_async/receiver_async.py @@ -18,7 +18,7 @@ class AsyncReceiver(Receiver): Implements the async API of a Receiver. """ - def __init__(self, client, source, offset=None, prefetch=300, epoch=None, loop=None): # pylint: disable=super-init-not-called + def __init__(self, client, source, offset=None, prefetch=300, epoch=None, keep_alive=None, loop=None): # pylint: disable=super-init-not-called """ Instantiate an async receiver. @@ -39,6 +39,7 @@ def __init__(self, client, source, offset=None, prefetch=300, epoch=None, loop=N self.offset = offset self.prefetch = prefetch self.epoch = epoch + self.keep_alive = keep_alive self.retry_policy = errors.ErrorPolicy(max_retries=3, on_error=_error_handler) self.redirected = None self.error = None @@ -56,7 +57,7 @@ def __init__(self, client, source, offset=None, prefetch=300, epoch=None, loop=N link_properties=self.properties, timeout=self.timeout, error_policy=self.retry_policy, - keep_alive_interval=30, + keep_alive_interval=self.keep_alive, loop=self.loop) async def open_async(self): @@ -85,7 +86,7 @@ async def open_async(self): link_properties=self.properties, timeout=self.timeout, error_policy=self.retry_policy, - keep_alive_interval=30, + keep_alive_interval=self.keep_alive, loop=self.loop) await self._handler.open_async() while not await self.has_started(): @@ -110,7 +111,7 @@ async def reconnect_async(self): link_properties=self.properties, timeout=self.timeout, error_policy=self.retry_policy, - keep_alive_interval=30, + keep_alive_interval=self.keep_alive, properties=self.client.create_properties(), loop=self.loop) await self._handler.open_async() diff --git a/azure/eventhub/_async/sender_async.py b/azure/eventhub/_async/sender_async.py index 42865f3..eec5b54 100644 --- a/azure/eventhub/_async/sender_async.py +++ b/azure/eventhub/_async/sender_async.py @@ -17,7 +17,7 @@ class AsyncSender(Sender): Implements the async API of a Sender. """ - def __init__(self, client, target, partition=None, loop=None): # pylint: disable=super-init-not-called + def __init__(self, client, target, partition=None, keep_alive=None, loop=None): # pylint: disable=super-init-not-called """ Instantiate an EventHub event SenderAsync handler. @@ -31,6 +31,7 @@ def __init__(self, client, target, partition=None, loop=None): # pylint: disabl self.client = client self.target = target self.partition = partition + self.keep_alive = keep_alive self.retry_policy = errors.ErrorPolicy(max_retries=3, on_error=_error_handler) self.redirected = None self.error = None @@ -42,7 +43,7 @@ def __init__(self, client, target, partition=None, loop=None): # pylint: disabl debug=self.client.debug, msg_timeout=Sender.TIMEOUT, error_policy=self.retry_policy, - keep_alive_interval=30, + keep_alive_interval=self.keep_alive, properties=self.client.create_properties(), loop=self.loop) self._outcome = None @@ -65,7 +66,7 @@ async def open_async(self): debug=self.client.debug, msg_timeout=Sender.TIMEOUT, error_policy=self.retry_policy, - keep_alive_interval=30, + keep_alive_interval=self.keep_alive, properties=self.client.create_properties(), loop=self.loop) await self._handler.open_async() @@ -85,7 +86,7 @@ async def reconnect_async(self): debug=self.client.debug, msg_timeout=Sender.TIMEOUT, error_policy=self.retry_policy, - keep_alive_interval=30, + keep_alive_interval=self.keep_alive, properties=self.client.create_properties(), loop=self.loop) await self._handler.open_async() diff --git a/azure/eventhub/client.py b/azure/eventhub/client.py index 9c57cbd..37d9e56 100644 --- a/azure/eventhub/client.py +++ b/azure/eventhub/client.py @@ -303,7 +303,7 @@ def get_eventhub_info(self): finally: mgmt_client.close() - def add_receiver(self, consumer_group, partition, offset=None, prefetch=300, operation=None): + def add_receiver(self, consumer_group, partition, offset=None, prefetch=300, operation=None, keep_alive=30): """ Add a receiver to the client for a particular consumer group and partition. @@ -323,11 +323,11 @@ def add_receiver(self, consumer_group, partition, offset=None, prefetch=300, ope path = self.address.path + operation if operation else self.address.path source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( self.address.hostname, path, consumer_group, partition) - handler = Receiver(self, source_url, offset=offset, prefetch=prefetch) + handler = Receiver(self, source_url, offset=offset, prefetch=prefetch, keep_alive=keep_alive) self.clients.append(handler) return handler - def add_epoch_receiver(self, consumer_group, partition, epoch, prefetch=300, operation=None): + def add_epoch_receiver(self, consumer_group, partition, epoch, prefetch=300, operation=None, keep_alive=30): """ Add a receiver to the client with an epoch value. Only a single epoch receiver can connect to a partition at any given time - additional epoch receivers must have @@ -350,11 +350,11 @@ def add_epoch_receiver(self, consumer_group, partition, epoch, prefetch=300, ope path = self.address.path + operation if operation else self.address.path source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( self.address.hostname, path, consumer_group, partition) - handler = Receiver(self, source_url, prefetch=prefetch, epoch=epoch) + handler = Receiver(self, source_url, prefetch=prefetch, epoch=epoch, keep_alive=keep_alive) self.clients.append(handler) return handler - def add_sender(self, partition=None, operation=None): + def add_sender(self, partition=None, operation=None, keep_alive=30): """ Add a sender to the client to send ~azure.eventhub.common.EventData object to an EventHub. @@ -371,6 +371,6 @@ def add_sender(self, partition=None, operation=None): target = "amqps://{}{}".format(self.address.hostname, self.address.path) if operation: target = target + operation - handler = Sender(self, target, partition=partition) + handler = Sender(self, target, partition=partition, keep_alive=keep_alive) self.clients.append(handler) return handler diff --git a/azure/eventhub/receiver.py b/azure/eventhub/receiver.py index 49a15ce..ef8383b 100644 --- a/azure/eventhub/receiver.py +++ b/azure/eventhub/receiver.py @@ -16,7 +16,7 @@ class Receiver: timeout = 0 _epoch = b'com.microsoft:epoch' - def __init__(self, client, source, offset=None, prefetch=300, epoch=None): + def __init__(self, client, source, offset=None, prefetch=300, epoch=None, keep_alive=None): """ Instantiate a receiver. @@ -35,6 +35,7 @@ def __init__(self, client, source, offset=None, prefetch=300, epoch=None): self.offset = offset self.prefetch = prefetch self.epoch = epoch + self.keep_alive = keep_alive self.retry_policy = errors.ErrorPolicy(max_retries=3, on_error=_error_handler) self.properties = None self.redirected = None @@ -52,7 +53,7 @@ def __init__(self, client, source, offset=None, prefetch=300, epoch=None): link_properties=self.properties, timeout=self.timeout, error_policy=self.retry_policy, - keep_alive_interval=30, + keep_alive_interval=self.keep_alive, properties=self.client.create_properties()) def open(self): @@ -81,7 +82,7 @@ def open(self): link_properties=self.properties, timeout=self.timeout, error_policy=self.retry_policy, - keep_alive_interval=30, + keep_alive_interval=self.keep_alive, properties=self.client.create_properties()) self._handler.open() while not self.has_started(): @@ -106,7 +107,7 @@ def reconnect(self): link_properties=self.properties, timeout=self.timeout, error_policy=self.retry_policy, - keep_alive_interval=30, + keep_alive_interval=self.keep_alive, properties=self.client.create_properties()) self._handler.open() while not self.has_started(): diff --git a/azure/eventhub/sender.py b/azure/eventhub/sender.py index ff358d0..cca271e 100644 --- a/azure/eventhub/sender.py +++ b/azure/eventhub/sender.py @@ -15,7 +15,7 @@ class Sender: """ TIMEOUT = 60.0 - def __init__(self, client, target, partition=None): + def __init__(self, client, target, partition=None, keep_alive=None): """ Instantiate an EventHub event Sender handler. @@ -29,6 +29,7 @@ def __init__(self, client, target, partition=None): self.partition = partition self.redirected = None self.error = None + self.keep_alive = keep_alive self.retry_policy = errors.ErrorPolicy(max_retries=3, on_error=_error_handler) if partition: self.target += "/Partitions/" + partition @@ -38,7 +39,7 @@ def __init__(self, client, target, partition=None): debug=self.client.debug, msg_timeout=Sender.TIMEOUT, error_policy=self.retry_policy, - keep_alive_interval=30, + keep_alive_interval=self.keep_alive, properties=self.client.create_properties()) self._outcome = None self._condition = None @@ -60,7 +61,7 @@ def open(self): debug=self.client.debug, msg_timeout=Sender.TIMEOUT, error_policy=self.retry_policy, - keep_alive_interval=30, + keep_alive_interval=self.keep_alive, properties=self.client.create_properties()) self._handler.open() while not self.has_started(): @@ -79,7 +80,7 @@ def reconnect(self): debug=self.client.debug, msg_timeout=Sender.TIMEOUT, error_policy=self.retry_policy, - keep_alive_interval=30, + keep_alive_interval=self.keep_alive, properties=self.client.create_properties()) self._handler.open() self._handler._pending_messages = unsent_events diff --git a/azure/eventprocessorhost/azure_storage_checkpoint_manager.py b/azure/eventprocessorhost/azure_storage_checkpoint_manager.py index a631fdb..001eafa 100644 --- a/azure/eventprocessorhost/azure_storage_checkpoint_manager.py +++ b/azure/eventprocessorhost/azure_storage_checkpoint_manager.py @@ -131,7 +131,7 @@ async def update_checkpoint_async(self, lease, checkpoint): new_lease.with_source(lease) new_lease.offset = checkpoint.offset new_lease.sequence_number = checkpoint.sequence_number - await self.update_lease_async(new_lease) + return await self.update_lease_async(new_lease) async def delete_checkpoint_async(self, partition_id): """ @@ -331,7 +331,7 @@ async def acquire_lease_async(self, lease): lease.owner = self.host.host_name lease.increment_epoch() # check if this solves the issue - await self.update_lease_async(lease) + retval = await self.update_lease_async(lease) except Exception as err: # pylint: disable=broad-except _logger.error("Failed to acquire lease {!r} {} {}".format( err, partition_id, lease.token)) @@ -361,7 +361,7 @@ async def renew_lease_async(self, lease): timeout=self.lease_duration)) except Exception as err: # pylint: disable=broad-except if "LeaseIdMismatchWithLeaseOperation" in str(err): - _logger.info("LeaseLost") + _logger.info("LeaseLost on partition {}".format(lease.partition_id)) else: _logger.error("Failed to renew lease on partition {} with token {} {!r}".format( lease.partition_id, lease.token, err)) diff --git a/azure/eventprocessorhost/eh_partition_pump.py b/azure/eventprocessorhost/eh_partition_pump.py index 4801a25..1b4d2dc 100644 --- a/azure/eventprocessorhost/eh_partition_pump.py +++ b/azure/eventprocessorhost/eh_partition_pump.py @@ -73,6 +73,7 @@ async def open_clients_async(self): self.partition_context.partition_id, Offset(self.partition_context.offset), prefetch=self.host.eph_options.prefetch_count, + keep_alive=None, loop=self.loop) self.partition_receiver = PartitionReceiver(self) @@ -95,7 +96,12 @@ async def on_closing_async(self, reason): :type reason: str """ self.partition_receiver.eh_partition_pump.set_pump_status("Errored") - await self.running + try: + await self.running + except TypeError: + _logger.debug("No partition pump running.") + except Exception as err: + _logger.info("Error on closing partition pump: {!r}".format(err)) await self.clean_up_clients_async() diff --git a/azure/eventprocessorhost/partition_context.py b/azure/eventprocessorhost/partition_context.py index b21514b..33cc566 100644 --- a/azure/eventprocessorhost/partition_context.py +++ b/azure/eventprocessorhost/partition_context.py @@ -115,7 +115,9 @@ async def persist_checkpoint_async(self, checkpoint): _logger.info("persisting checkpoint {}".format(checkpoint.__dict__)) await self.host.storage_manager.create_checkpoint_if_not_exists_async(checkpoint.partition_id) - await self.host.storage_manager.update_checkpoint_async(self.lease, checkpoint) + if not await self.host.storage_manager.update_checkpoint_async(self.lease, checkpoint): + _logger.error("Failed to persist checkpoint for partition: {}".format(self.partition_id)) + raise Exception("failed to persist checkpoint") self.lease.offset = checkpoint.offset self.lease.sequence_number = checkpoint.sequence_number else: diff --git a/azure/eventprocessorhost/partition_manager.py b/azure/eventprocessorhost/partition_manager.py index 7025fe0..c9c927c 100644 --- a/azure/eventprocessorhost/partition_manager.py +++ b/azure/eventprocessorhost/partition_manager.py @@ -190,6 +190,7 @@ async def run_loop_async(self): self.host.guid, partition_id)) await self.check_and_add_pump_async(partition_id, updated_lease) else: + _logger.debug("Removing pump due to lost lease.") await self.remove_pump_async(partition_id, "LeaseLost") except Exception as err: # pylint: disable=broad-except _logger.error("Failed to update lease {!r}".format(err)) From a9d638e97cdfcfd5cb6be613c755e65b519c0d67 Mon Sep 17 00:00:00 2001 From: annatisch Date: Sat, 4 Aug 2018 14:21:00 -0700 Subject: [PATCH 25/52] Made reconnect configurable --- azure/eventhub/_async/__init__.py | 12 +++++----- azure/eventhub/_async/receiver_async.py | 32 +++++++++++++++++++++---- azure/eventhub/_async/sender_async.py | 31 ++++++++++++++++++------ azure/eventhub/client.py | 12 +++++----- azure/eventhub/receiver.py | 23 ++++++++++++++---- azure/eventhub/sender.py | 32 +++++++++++++++++++------ tests/test_iothub_receive_async.py | 10 ++++---- 7 files changed, 110 insertions(+), 42 deletions(-) diff --git a/azure/eventhub/_async/__init__.py b/azure/eventhub/_async/__init__.py index 126fdc8..7c1d095 100644 --- a/azure/eventhub/_async/__init__.py +++ b/azure/eventhub/_async/__init__.py @@ -164,7 +164,7 @@ async def get_eventhub_info_async(self): finally: await mgmt_client.close_async() - def add_async_receiver(self, consumer_group, partition, offset=None, prefetch=300, operation=None, keep_alive=30, loop=None): + def add_async_receiver(self, consumer_group, partition, offset=None, prefetch=300, operation=None, keep_alive=30, auto_reconnect=True, loop=None): """ Add an async receiver to the client for a particular consumer group and partition. @@ -184,11 +184,11 @@ def add_async_receiver(self, consumer_group, partition, offset=None, prefetch=30 path = self.address.path + operation if operation else self.address.path source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( self.address.hostname, path, consumer_group, partition) - handler = AsyncReceiver(self, source_url, offset=offset, prefetch=prefetch, keep_alive=keep_alive, loop=loop) + handler = AsyncReceiver(self, source_url, offset=offset, prefetch=prefetch, keep_alive=keep_alive, auto_reconnect=auto_reconnect, loop=loop) self.clients.append(handler) return handler - def add_async_epoch_receiver(self, consumer_group, partition, epoch, prefetch=300, operation=None, keep_alive=30, loop=None): + def add_async_epoch_receiver(self, consumer_group, partition, epoch, prefetch=300, operation=None, keep_alive=30, auto_reconnect=True, loop=None): """ Add an async receiver to the client with an epoch value. Only a single epoch receiver can connect to a partition at any given time - additional epoch receivers must have @@ -211,11 +211,11 @@ def add_async_epoch_receiver(self, consumer_group, partition, epoch, prefetch=30 path = self.address.path + operation if operation else self.address.path source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( self.address.hostname, path, consumer_group, partition) - handler = AsyncReceiver(self, source_url, prefetch=prefetch, epoch=epoch, keep_alive=keep_alive, loop=loop) + handler = AsyncReceiver(self, source_url, prefetch=prefetch, epoch=epoch, keep_alive=keep_alive, auto_reconnect=auto_reconnect, loop=loop) self.clients.append(handler) return handler - def add_async_sender(self, partition=None, operation=None, keep_alive=30, loop=None): + def add_async_sender(self, partition=None, operation=None, keep_alive=30, auto_reconnect=True, loop=None): """ Add an async sender to the client to send ~azure.eventhub.common.EventData object to an EventHub. @@ -232,6 +232,6 @@ def add_async_sender(self, partition=None, operation=None, keep_alive=30, loop=N target = "amqps://{}{}".format(self.address.hostname, self.address.path) if operation: target = target + operation - handler = AsyncSender(self, target, partition=partition, keep_alive=keep_alive, loop=loop) + handler = AsyncSender(self, target, partition=partition, keep_alive=keep_alive, auto_reconnect=auto_reconnect, loop=loop) self.clients.append(handler) return handler diff --git a/azure/eventhub/_async/receiver_async.py b/azure/eventhub/_async/receiver_async.py index 22166d9..6af3d2b 100644 --- a/azure/eventhub/_async/receiver_async.py +++ b/azure/eventhub/_async/receiver_async.py @@ -4,6 +4,8 @@ # -------------------------------------------------------------------------------------------- import asyncio +import uuid +import logging from uamqp import errors, types from uamqp import ReceiveClientAsync, Source @@ -12,13 +14,15 @@ from azure.eventhub.receiver import Receiver from azure.eventhub.common import _error_handler +log = logging.getLogger(__name__) + class AsyncReceiver(Receiver): """ Implements the async API of a Receiver. """ - def __init__(self, client, source, offset=None, prefetch=300, epoch=None, keep_alive=None, loop=None): # pylint: disable=super-init-not-called + def __init__(self, client, source, offset=None, prefetch=300, epoch=None, keep_alive=None, auto_reconnect=True, loop=None): # pylint: disable=super-init-not-called """ Instantiate an async receiver. @@ -40,10 +44,13 @@ def __init__(self, client, source, offset=None, prefetch=300, epoch=None, keep_a self.prefetch = prefetch self.epoch = epoch self.keep_alive = keep_alive + self.auto_reconnect = auto_reconnect self.retry_policy = errors.ErrorPolicy(max_retries=3, on_error=_error_handler) self.redirected = None self.error = None self.properties = None + partition = self.source.split('/')[-1] + self.name = "EHReceiver-{}-partition{}".format(uuid.uuid4(), partition) source = Source(self.source) if self.offset is not None: source.set_filter(self.offset.selector()) @@ -58,6 +65,8 @@ def __init__(self, client, source, offset=None, prefetch=300, epoch=None, keep_a timeout=self.timeout, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, + client_name=self.name, + properties=self.client.create_properties(), loop=self.loop) async def open_async(self): @@ -87,6 +96,8 @@ async def open_async(self): timeout=self.timeout, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, + client_name=self.name, + properties=self.client.create_properties(), loop=self.loop) await self._handler.open_async() while not await self.has_started(): @@ -112,6 +123,7 @@ async def reconnect_async(self): timeout=self.timeout, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, + client_name=self.name, properties=self.client.create_properties(), loop=self.loop) await self._handler.open_async() @@ -190,17 +202,27 @@ async def receive(self, max_batch_size=None, timeout=None): data_batch.append(event_data) return data_batch except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry: + if shutdown.action.retry and self.auto_reconnect: + log.info("AsyncReceiver detached. Attempting reconnect.") await self.reconnect_async() return data_batch else: + log.info("AsyncReceiver detached. Shutting down.") + error = EventHubError(str(shutdown), shutdown) + await self.close_async(exception=error) + raise error + except errors.MessageHandlerError as shutdown: + if self.auto_reconnect: + log.info("AsyncReceiver detached. Attempting reconnect.") + await self.reconnect_async() + return data_batch + else: + log.info("AsyncReceiver detached. Shutting down.") error = EventHubError(str(shutdown), shutdown) await self.close_async(exception=error) raise error - except errors.MessageHandlerError: - await self.reconnect_async() - return data_batch except Exception as e: + log.info("Unexpected error occurred ({}). Shutting down.".format(e)) error = EventHubError("Receive failed: {}".format(e)) await self.close_async(exception=error) raise error diff --git a/azure/eventhub/_async/sender_async.py b/azure/eventhub/_async/sender_async.py index eec5b54..7dee78e 100644 --- a/azure/eventhub/_async/sender_async.py +++ b/azure/eventhub/_async/sender_async.py @@ -3,6 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- +import uuid import asyncio from uamqp import constants, errors @@ -17,7 +18,7 @@ class AsyncSender(Sender): Implements the async API of a Sender. """ - def __init__(self, client, target, partition=None, keep_alive=None, loop=None): # pylint: disable=super-init-not-called + def __init__(self, client, target, partition=None, keep_alive=None, auto_reconnect=True, loop=None): # pylint: disable=super-init-not-called """ Instantiate an EventHub event SenderAsync handler. @@ -32,11 +33,14 @@ def __init__(self, client, target, partition=None, keep_alive=None, loop=None): self.target = target self.partition = partition self.keep_alive = keep_alive + self.auto_reconnect = auto_reconnect self.retry_policy = errors.ErrorPolicy(max_retries=3, on_error=_error_handler) + self.name = "EHSender-{}".format(uuid.uuid4()) self.redirected = None self.error = None if partition: self.target += "/Partitions/" + partition + self.name += "-partition{}".format(partition) self._handler = SendClientAsync( self.target, auth=self.client.get_auth(), @@ -44,6 +48,7 @@ def __init__(self, client, target, partition=None, keep_alive=None, loop=None): msg_timeout=Sender.TIMEOUT, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, + client_name=self.name, properties=self.client.create_properties(), loop=self.loop) self._outcome = None @@ -67,6 +72,7 @@ async def open_async(self): msg_timeout=Sender.TIMEOUT, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, + client_name=self.name, properties=self.client.create_properties(), loop=self.loop) await self._handler.open_async() @@ -87,6 +93,7 @@ async def reconnect_async(self): msg_timeout=Sender.TIMEOUT, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, + client_name=self.name, properties=self.client.create_properties(), loop=self.loop) await self._handler.open_async() @@ -159,14 +166,19 @@ async def send(self, event_data): if self._outcome != constants.MessageSendResult.Ok: raise Sender._error(self._outcome, self._condition) except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry: + if shutdown.action.retry and self.auto_reconnect: + await self.reconnect_async() + else: + error = EventHubError(str(shutdown), shutdown) + await self.close_async(exception=error) + raise error + except errors.MessageHandlerError as shutdown: + if self.auto_reconnect: await self.reconnect_async() else: error = EventHubError(str(shutdown), shutdown) await self.close_async(exception=error) raise error - except errors.MessageHandlerError: - await self.reconnect_async() except Exception as e: error = EventHubError("Send failed: {}".format(e)) await self.close_async(exception=error) @@ -183,13 +195,18 @@ async def wait_async(self): try: await self._handler.wait_async() except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry: + if shutdown.action.retry and self.auto_reconnect: + await self.reconnect_async() + else: + error = EventHubError(str(shutdown), shutdown) + await self.close_async(exception=error) + raise error + except errors.MessageHandlerError as shutdown: + if self.auto_reconnect: await self.reconnect_async() else: error = EventHubError(str(shutdown), shutdown) await self.close_async(exception=error) raise error - except errors.MessageHandlerError: - await self.reconnect_async() except Exception as e: raise EventHubError("Send failed: {}".format(e)) diff --git a/azure/eventhub/client.py b/azure/eventhub/client.py index 37d9e56..7476e3f 100644 --- a/azure/eventhub/client.py +++ b/azure/eventhub/client.py @@ -303,7 +303,7 @@ def get_eventhub_info(self): finally: mgmt_client.close() - def add_receiver(self, consumer_group, partition, offset=None, prefetch=300, operation=None, keep_alive=30): + def add_receiver(self, consumer_group, partition, offset=None, prefetch=300, operation=None, keep_alive=30, auto_reconnect=True): """ Add a receiver to the client for a particular consumer group and partition. @@ -323,11 +323,11 @@ def add_receiver(self, consumer_group, partition, offset=None, prefetch=300, ope path = self.address.path + operation if operation else self.address.path source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( self.address.hostname, path, consumer_group, partition) - handler = Receiver(self, source_url, offset=offset, prefetch=prefetch, keep_alive=keep_alive) + handler = Receiver(self, source_url, offset=offset, prefetch=prefetch, keep_alive=keep_alive, auto_reconnect=auto_reconnect) self.clients.append(handler) return handler - def add_epoch_receiver(self, consumer_group, partition, epoch, prefetch=300, operation=None, keep_alive=30): + def add_epoch_receiver(self, consumer_group, partition, epoch, prefetch=300, operation=None, keep_alive=30, auto_reconnect=True): """ Add a receiver to the client with an epoch value. Only a single epoch receiver can connect to a partition at any given time - additional epoch receivers must have @@ -350,11 +350,11 @@ def add_epoch_receiver(self, consumer_group, partition, epoch, prefetch=300, ope path = self.address.path + operation if operation else self.address.path source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( self.address.hostname, path, consumer_group, partition) - handler = Receiver(self, source_url, prefetch=prefetch, epoch=epoch, keep_alive=keep_alive) + handler = Receiver(self, source_url, prefetch=prefetch, epoch=epoch, keep_alive=keep_alive, auto_reconnect=auto_reconnect) self.clients.append(handler) return handler - def add_sender(self, partition=None, operation=None, keep_alive=30): + def add_sender(self, partition=None, operation=None, keep_alive=30, auto_reconnect=True): """ Add a sender to the client to send ~azure.eventhub.common.EventData object to an EventHub. @@ -371,6 +371,6 @@ def add_sender(self, partition=None, operation=None, keep_alive=30): target = "amqps://{}{}".format(self.address.hostname, self.address.path) if operation: target = target + operation - handler = Sender(self, target, partition=partition, keep_alive=keep_alive) + handler = Sender(self, target, partition=partition, keep_alive=keep_alive, auto_reconnect=auto_reconnect) self.clients.append(handler) return handler diff --git a/azure/eventhub/receiver.py b/azure/eventhub/receiver.py index ef8383b..e0fed9e 100644 --- a/azure/eventhub/receiver.py +++ b/azure/eventhub/receiver.py @@ -3,6 +3,8 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- +import uuid + from uamqp import types, errors from uamqp import ReceiveClient, Source @@ -16,7 +18,7 @@ class Receiver: timeout = 0 _epoch = b'com.microsoft:epoch' - def __init__(self, client, source, offset=None, prefetch=300, epoch=None, keep_alive=None): + def __init__(self, client, source, offset=None, prefetch=300, epoch=None, keep_alive=None, auto_reconnect=True): """ Instantiate a receiver. @@ -36,10 +38,13 @@ def __init__(self, client, source, offset=None, prefetch=300, epoch=None, keep_a self.prefetch = prefetch self.epoch = epoch self.keep_alive = keep_alive + self.auto_reconnect = True self.retry_policy = errors.ErrorPolicy(max_retries=3, on_error=_error_handler) self.properties = None self.redirected = None self.error = None + partition = self.source.split('/')[-1] + self.name = "EHReceiver-{}-partition{}".format(uuid.uuid4(), partition) source = Source(self.source) if self.offset is not None: source.set_filter(self.offset.selector()) @@ -54,6 +59,7 @@ def __init__(self, client, source, offset=None, prefetch=300, epoch=None, keep_a timeout=self.timeout, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, + client_name=self.name, properties=self.client.create_properties()) def open(self): @@ -83,6 +89,7 @@ def open(self): timeout=self.timeout, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, + client_name=self.name, properties=self.client.create_properties()) self._handler.open() while not self.has_started(): @@ -108,6 +115,7 @@ def reconnect(self): timeout=self.timeout, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, + client_name=self.name, properties=self.client.create_properties()) self._handler.open() while not self.has_started(): @@ -205,16 +213,21 @@ def receive(self, max_batch_size=None, timeout=None): data_batch.append(event_data) return data_batch except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry: + if shutdown.action.retry and self.auto_reconnect: + self.reconnect() + return data_batch + else: + error = EventHubError(str(shutdown), shutdown) + self.close(exception=error) + raise error + except errors.MessageHandlerError as shutdown: + if self.auto_reconnect: self.reconnect() return data_batch else: error = EventHubError(str(shutdown), shutdown) self.close(exception=error) raise error - except errors.MessageHandlerError: - self.reconnect() - return data_batch except Exception as e: error = EventHubError("Receive failed: {}".format(e)) self.close(exception=error) diff --git a/azure/eventhub/sender.py b/azure/eventhub/sender.py index cca271e..b59ed70 100644 --- a/azure/eventhub/sender.py +++ b/azure/eventhub/sender.py @@ -3,6 +3,8 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- +import uuid + from uamqp import constants, errors from uamqp import SendClient @@ -15,7 +17,7 @@ class Sender: """ TIMEOUT = 60.0 - def __init__(self, client, target, partition=None, keep_alive=None): + def __init__(self, client, target, partition=None, keep_alive=None, auto_reconnect=True): """ Instantiate an EventHub event Sender handler. @@ -30,9 +32,12 @@ def __init__(self, client, target, partition=None, keep_alive=None): self.redirected = None self.error = None self.keep_alive = keep_alive + self.auto_reconnect = auto_reconnect self.retry_policy = errors.ErrorPolicy(max_retries=3, on_error=_error_handler) + self.name = "EHSender-{}".format(uuid.uuid4()) if partition: self.target += "/Partitions/" + partition + self.name += "-partition{}".format(partition) self._handler = SendClient( self.target, auth=self.client.get_auth(), @@ -40,6 +45,7 @@ def __init__(self, client, target, partition=None, keep_alive=None): msg_timeout=Sender.TIMEOUT, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, + client_name=self.name, properties=self.client.create_properties()) self._outcome = None self._condition = None @@ -62,6 +68,7 @@ def open(self): msg_timeout=Sender.TIMEOUT, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, + client_name=self.name, properties=self.client.create_properties()) self._handler.open() while not self.has_started(): @@ -81,6 +88,7 @@ def reconnect(self): msg_timeout=Sender.TIMEOUT, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, + client_name=self.name, properties=self.client.create_properties()) self._handler.open() self._handler._pending_messages = unsent_events @@ -166,14 +174,19 @@ def send(self, event_data): self.close(exception=error) raise error except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry: + if shutdown.action.retry and self.auto_reconnect: + self.reconnect() + else: + error = EventHubError(str(shutdown), shutdown) + self.close(exception=error) + raise error + except errors.MessageHandlerError as shutdown: + if self.auto_reconnect: self.reconnect() else: error = EventHubError(str(shutdown), shutdown) self.close(exception=error) raise error - except errors.MessageHandlerError: - self.reconnect() except Exception as e: error = EventHubError("Send failed: {}".format(e)) self.close(exception=error) @@ -208,14 +221,19 @@ def wait(self): try: self._handler.wait() except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry: + if shutdown.action.retry and self.auto_reconnect: + self.reconnect() + else: + error = EventHubError(str(shutdown), shutdown) + self.close(exception=error) + raise error + except errors.MessageHandlerError as shutdown: + if self.auto_reconnect: self.reconnect() else: error = EventHubError(str(shutdown), shutdown) self.close(exception=error) raise error - except errors.MessageHandlerError: - self.reconnect() except Exception as e: raise EventHubError("Send failed: {}".format(e)) diff --git a/tests/test_iothub_receive_async.py b/tests/test_iothub_receive_async.py index d26c00c..66bea90 100644 --- a/tests/test_iothub_receive_async.py +++ b/tests/test_iothub_receive_async.py @@ -18,9 +18,7 @@ async def pump(receiver, sleep=None): if sleep: await asyncio.sleep(sleep) batch = await receiver.receive(timeout=1) - while batch: - messages += len(batch) - batch = await receiver.receive(timeout=1) + messages += len(batch) return messages @@ -42,11 +40,11 @@ async def test_iothub_receive_multiple_async(iot_connection_str): try: receivers = [] for p in partitions: - receivers.append(client.add_async_receiver("$default", p, prefetch=1000, operation='/messages/events')) + receivers.append(client.add_async_receiver("$default", p, prefetch=10, operation='/messages/events')) await client.run_async() outputs = await asyncio.gather(*[pump(r) for r in receivers]) - assert isinstance(outputs[0], int) and outputs[0] == 0 - assert isinstance(outputs[1], int) and outputs[1] == 0 + assert isinstance(outputs[0], int) and outputs[0] <= 10 + assert isinstance(outputs[1], int) and outputs[1] <= 10 finally: await client.stop_async() From 4a82e105c47507891084edd7856c86d36eaa5b41 Mon Sep 17 00:00:00 2001 From: annatisch Date: Sat, 4 Aug 2018 14:35:12 -0700 Subject: [PATCH 26/52] Added more EPH options --- azure/eventprocessorhost/eh_partition_pump.py | 3 +- azure/eventprocessorhost/eph.py | 34 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/azure/eventprocessorhost/eh_partition_pump.py b/azure/eventprocessorhost/eh_partition_pump.py index 1b4d2dc..8260bac 100644 --- a/azure/eventprocessorhost/eh_partition_pump.py +++ b/azure/eventprocessorhost/eh_partition_pump.py @@ -73,7 +73,8 @@ async def open_clients_async(self): self.partition_context.partition_id, Offset(self.partition_context.offset), prefetch=self.host.eph_options.prefetch_count, - keep_alive=None, + keep_alive=self.host.eph_options.keep_alive_interval, + auto_reconnect=self.host.eph_options.auto_reconnect_on_error, loop=self.loop) self.partition_receiver = PartitionReceiver(self) diff --git a/azure/eventprocessorhost/eph.py b/azure/eventprocessorhost/eph.py index 7c7541e..2e464e1 100644 --- a/azure/eventprocessorhost/eph.py +++ b/azure/eventprocessorhost/eph.py @@ -64,6 +64,38 @@ async def close_async(self): class EPHOptions: """ Class that contains default and overidable EPH option. + + :ivar max_batch_size: The maximum number of events retrieved for processing + at a time. This value must be less than or equal to the prefetch count. The actual + number of events returned for processing may be any number up to the maximum. + The default value is 10. + :vartype max_batch_size: int + :ivar prefetch_count: The number of events to fetch from the service in advance of + processing. The default value is 300. + :vartype prefetch_count: int + :ivar receive_timeout: The length of time a single partition receiver will wait in + order to receive a batch of events. Default is 60 seconds. + :vartype receive_timeout: int + :ivar release_pump_on_timeout: Whether to shutdown an individual partition receiver if + no events were received in the specified timeout. Shutting down the pump will release + the lease to allow it to be picked up by another host. Default is False. + :vartype release_pump_on_timeout: bool + :ivar initial_offset_provider: The initial event offset to receive from if no persisted + offset is found. Default is "-1" (i.e. from the first event available). + :vartype initial_offset_provider: str + :ivar debug_trace: Whether to emit the network traffic in the logs. In order to view + these events the logger must be configured to track "uamqp". Default is False. + :vartype debug_trace: bool + :ivar http_proxy: HTTP proxy configuration. This should be a dictionary with + the following keys present: 'proxy_hostname' and 'proxy_port'. Additional optional + keys are 'username' and 'password'. + :vartype http_proxy: dict + :ivar keep_alive_interval: The time in seconds between asynchronously pinging a receiver + connection to keep it alive during inactivity. Default is None - i.e. no connection pinging. + :vartype keep_alive_interval: int + :ivar auto_reconnect_on_error: Whether to automatically attempt to reconnect a receiver + connection if it is detach from the service with a retryable error. Default is True. + :vartype auto_reconnect_on_error: bool """ def __init__(self): @@ -74,3 +106,5 @@ def __init__(self): self.initial_offset_provider = "-1" self.debug_trace = False self.http_proxy = None + self.keep_alive_interval = None + self.auto_reconnect_on_error = True From 01442b648fc0dce53701e4e96626d7f8656099cd Mon Sep 17 00:00:00 2001 From: annatisch Date: Tue, 7 Aug 2018 09:08:55 -0700 Subject: [PATCH 27/52] Bumped version --- HISTORY.rst | 16 ++++++++++++++ azure/eventhub/__init__.py | 2 +- azure/eventhub/_async/__init__.py | 21 ++++++++++++++----- azure/eventhub/_async/receiver_async.py | 4 +++- azure/eventhub/client.py | 16 ++++++++++---- azure/eventhub/receiver.py | 2 +- azure/eventprocessorhost/eh_partition_pump.py | 1 + setup.py | 2 +- 8 files changed, 51 insertions(+), 13 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 89db3a3..d60b724 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,22 @@ Release History =============== +0.2.0 (2018-08-06) +++++++++++++++++++ + +- Stability improvements for EPH. +- Updated uAMQP version. +- Added new configuration options for Sender and Receiver; `keep_alive` and `auto_reconnect`. + These flags have been added to the following: + + - `EventHubClient.add_receiver` + - `EventHubClient.add_sender` + - `EventHubClientAsync.add_async_receiver` + - `EventHubClientAsync.add_async_sender` + - `EPHOptions.keey_alive_interval` + - `EPHOptions.auto_reconnect_on_error` + + 0.2.0rc2 (2018-07-29) +++++++++++++++++++++ diff --git a/azure/eventhub/__init__.py b/azure/eventhub/__init__.py index 5acadea..ae780c2 100644 --- a/azure/eventhub/__init__.py +++ b/azure/eventhub/__init__.py @@ -3,7 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -__version__ = "0.2.0rc2" +__version__ = "0.2.0" from azure.eventhub.common import EventData, EventHubError, Offset from azure.eventhub.client import EventHubClient diff --git a/azure/eventhub/_async/__init__.py b/azure/eventhub/_async/__init__.py index 7c1d095..c4bcadf 100644 --- a/azure/eventhub/_async/__init__.py +++ b/azure/eventhub/_async/__init__.py @@ -77,6 +77,7 @@ async def _start_client_async(self, client): try: await client.open_async() except Exception as exp: # pylint: disable=broad-except + log.info("Encountered error while starting handler: {}".format(exp)) await client.close_async(exception=exp) async def _handle_redirect(self, redirects): @@ -164,7 +165,9 @@ async def get_eventhub_info_async(self): finally: await mgmt_client.close_async() - def add_async_receiver(self, consumer_group, partition, offset=None, prefetch=300, operation=None, keep_alive=30, auto_reconnect=True, loop=None): + def add_async_receiver( + self, consumer_group, partition, offset=None, prefetch=300, + operation=None, keep_alive=30, auto_reconnect=True, loop=None): """ Add an async receiver to the client for a particular consumer group and partition. @@ -184,11 +187,15 @@ def add_async_receiver(self, consumer_group, partition, offset=None, prefetch=30 path = self.address.path + operation if operation else self.address.path source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( self.address.hostname, path, consumer_group, partition) - handler = AsyncReceiver(self, source_url, offset=offset, prefetch=prefetch, keep_alive=keep_alive, auto_reconnect=auto_reconnect, loop=loop) + handler = AsyncReceiver( + self, source_url, offset=offset, prefetch=prefetch, + keep_alive=keep_alive, auto_reconnect=auto_reconnect, loop=loop) self.clients.append(handler) return handler - def add_async_epoch_receiver(self, consumer_group, partition, epoch, prefetch=300, operation=None, keep_alive=30, auto_reconnect=True, loop=None): + def add_async_epoch_receiver( + self, consumer_group, partition, epoch, prefetch=300, + operation=None, keep_alive=30, auto_reconnect=True, loop=None): """ Add an async receiver to the client with an epoch value. Only a single epoch receiver can connect to a partition at any given time - additional epoch receivers must have @@ -211,7 +218,9 @@ def add_async_epoch_receiver(self, consumer_group, partition, epoch, prefetch=30 path = self.address.path + operation if operation else self.address.path source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( self.address.hostname, path, consumer_group, partition) - handler = AsyncReceiver(self, source_url, prefetch=prefetch, epoch=epoch, keep_alive=keep_alive, auto_reconnect=auto_reconnect, loop=loop) + handler = AsyncReceiver( + self, source_url, prefetch=prefetch, epoch=epoch, + keep_alive=keep_alive, auto_reconnect=auto_reconnect, loop=loop) self.clients.append(handler) return handler @@ -232,6 +241,8 @@ def add_async_sender(self, partition=None, operation=None, keep_alive=30, auto_r target = "amqps://{}{}".format(self.address.hostname, self.address.path) if operation: target = target + operation - handler = AsyncSender(self, target, partition=partition, keep_alive=keep_alive, auto_reconnect=auto_reconnect, loop=loop) + handler = AsyncSender( + self, target, partition=partition, keep_alive=keep_alive, + auto_reconnect=auto_reconnect, loop=loop) self.clients.append(handler) return handler diff --git a/azure/eventhub/_async/receiver_async.py b/azure/eventhub/_async/receiver_async.py index 6af3d2b..1bf5c06 100644 --- a/azure/eventhub/_async/receiver_async.py +++ b/azure/eventhub/_async/receiver_async.py @@ -22,7 +22,9 @@ class AsyncReceiver(Receiver): Implements the async API of a Receiver. """ - def __init__(self, client, source, offset=None, prefetch=300, epoch=None, keep_alive=None, auto_reconnect=True, loop=None): # pylint: disable=super-init-not-called + def __init__( + self, client, source, offset=None, prefetch=300, epoch=None, + keep_alive=None, auto_reconnect=True, loop=None): # pylint: disable=super-init-not-called """ Instantiate an async receiver. diff --git a/azure/eventhub/client.py b/azure/eventhub/client.py index 7476e3f..08f07bd 100644 --- a/azure/eventhub/client.py +++ b/azure/eventhub/client.py @@ -303,7 +303,9 @@ def get_eventhub_info(self): finally: mgmt_client.close() - def add_receiver(self, consumer_group, partition, offset=None, prefetch=300, operation=None, keep_alive=30, auto_reconnect=True): + def add_receiver( + self, consumer_group, partition, offset=None, prefetch=300, + operation=None, keep_alive=30, auto_reconnect=True): """ Add a receiver to the client for a particular consumer group and partition. @@ -323,11 +325,15 @@ def add_receiver(self, consumer_group, partition, offset=None, prefetch=300, ope path = self.address.path + operation if operation else self.address.path source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( self.address.hostname, path, consumer_group, partition) - handler = Receiver(self, source_url, offset=offset, prefetch=prefetch, keep_alive=keep_alive, auto_reconnect=auto_reconnect) + handler = Receiver( + self, source_url, offset=offset, prefetch=prefetch, + keep_alive=keep_alive, auto_reconnect=auto_reconnect) self.clients.append(handler) return handler - def add_epoch_receiver(self, consumer_group, partition, epoch, prefetch=300, operation=None, keep_alive=30, auto_reconnect=True): + def add_epoch_receiver( + self, consumer_group, partition, epoch, prefetch=300, + operation=None, keep_alive=30, auto_reconnect=True): """ Add a receiver to the client with an epoch value. Only a single epoch receiver can connect to a partition at any given time - additional epoch receivers must have @@ -350,7 +356,9 @@ def add_epoch_receiver(self, consumer_group, partition, epoch, prefetch=300, ope path = self.address.path + operation if operation else self.address.path source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( self.address.hostname, path, consumer_group, partition) - handler = Receiver(self, source_url, prefetch=prefetch, epoch=epoch, keep_alive=keep_alive, auto_reconnect=auto_reconnect) + handler = Receiver( + self, source_url, prefetch=prefetch, epoch=epoch, + keep_alive=keep_alive, auto_reconnect=auto_reconnect) self.clients.append(handler) return handler diff --git a/azure/eventhub/receiver.py b/azure/eventhub/receiver.py index e0fed9e..90af41e 100644 --- a/azure/eventhub/receiver.py +++ b/azure/eventhub/receiver.py @@ -38,7 +38,7 @@ def __init__(self, client, source, offset=None, prefetch=300, epoch=None, keep_a self.prefetch = prefetch self.epoch = epoch self.keep_alive = keep_alive - self.auto_reconnect = True + self.auto_reconnect = auto_reconnect self.retry_policy = errors.ErrorPolicy(max_retries=3, on_error=_error_handler) self.properties = None self.redirected = None diff --git a/azure/eventprocessorhost/eh_partition_pump.py b/azure/eventprocessorhost/eh_partition_pump.py index 8260bac..31f677d 100644 --- a/azure/eventprocessorhost/eh_partition_pump.py +++ b/azure/eventprocessorhost/eh_partition_pump.py @@ -128,6 +128,7 @@ async def run(self): max_batch_size=self.max_batch_size, timeout=self.recieve_timeout) except Exception as e: # pylint: disable=broad-except + _logger.info("Error raised while attempting to receive messages: {}".format(e)) await self.process_error_async(e) else: if not msgs: diff --git a/setup.py b/setup.py index df46435..8efb8aa 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ zip_safe=False, packages=find_packages(exclude=["examples", "tests"]), install_requires=[ - 'uamqp~=0.2.0', + 'uamqp>=0.2.1,<0.3.0', 'msrestazure~=0.4.11', 'azure-common~=1.1', 'azure-storage~=0.36.0' From cb4ce1e2d17fba9b07b183ec5b99081cd36eb764 Mon Sep 17 00:00:00 2001 From: annatisch Date: Tue, 7 Aug 2018 09:19:08 -0700 Subject: [PATCH 28/52] Pylint fix --- azure/eventhub/_async/receiver_async.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure/eventhub/_async/receiver_async.py b/azure/eventhub/_async/receiver_async.py index 1bf5c06..4cf315c 100644 --- a/azure/eventhub/_async/receiver_async.py +++ b/azure/eventhub/_async/receiver_async.py @@ -22,9 +22,9 @@ class AsyncReceiver(Receiver): Implements the async API of a Receiver. """ - def __init__( + def __init__( # pylint: disable=super-init-not-called self, client, source, offset=None, prefetch=300, epoch=None, - keep_alive=None, auto_reconnect=True, loop=None): # pylint: disable=super-init-not-called + keep_alive=None, auto_reconnect=True, loop=None): """ Instantiate an async receiver. From 0c275728323fce201baad907aff2bfb9697da03e Mon Sep 17 00:00:00 2001 From: annatisch Date: Tue, 7 Aug 2018 09:22:50 -0700 Subject: [PATCH 29/52] Pylint fix --- azure/eventprocessorhost/eh_partition_pump.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure/eventprocessorhost/eh_partition_pump.py b/azure/eventprocessorhost/eh_partition_pump.py index 31f677d..368c2bb 100644 --- a/azure/eventprocessorhost/eh_partition_pump.py +++ b/azure/eventprocessorhost/eh_partition_pump.py @@ -101,7 +101,7 @@ async def on_closing_async(self, reason): await self.running except TypeError: _logger.debug("No partition pump running.") - except Exception as err: + except Exception as err: # pylint: disable=broad-except _logger.info("Error on closing partition pump: {!r}".format(err)) await self.clean_up_clients_async() From aeea0a752f0326164c3cd3828b38d40952013092 Mon Sep 17 00:00:00 2001 From: annatisch Date: Thu, 9 Aug 2018 11:17:29 -0700 Subject: [PATCH 30/52] Added send and auth timeouts --- azure/eventhub/_async/__init__.py | 17 ++++++++-- azure/eventhub/_async/sender_async.py | 29 ++++++++++++----- azure/eventhub/client.py | 47 ++++++++++++++++++++++++--- azure/eventhub/sender.py | 27 ++++++++++----- 4 files changed, 96 insertions(+), 24 deletions(-) diff --git a/azure/eventhub/_async/__init__.py b/azure/eventhub/_async/__init__.py index c4bcadf..6868462 100644 --- a/azure/eventhub/_async/__init__.py +++ b/azure/eventhub/_async/__init__.py @@ -58,7 +58,7 @@ def _create_auth(self, username=None, password=None): # pylint: disable=no-self return authentication.SASLPlain( self.address.hostname, username, password, http_proxy=self.http_proxy) return authentication.SASTokenAsync.from_shared_access_key( - self.auth_uri, username, password, timeout=60, http_proxy=self.http_proxy) + self.auth_uri, username, password, timeout=self.auth_timeout, http_proxy=self.http_proxy) async def _close_clients_async(self): """ @@ -79,6 +79,7 @@ async def _start_client_async(self, client): except Exception as exp: # pylint: disable=broad-except log.info("Encountered error while starting handler: {}".format(exp)) await client.close_async(exception=exp) + log.info("Finished closing failed handler") async def _handle_redirect(self, redirects): if len(redirects) != len(self.clients): @@ -224,7 +225,7 @@ def add_async_epoch_receiver( self.clients.append(handler) return handler - def add_async_sender(self, partition=None, operation=None, keep_alive=30, auto_reconnect=True, loop=None): + def add_async_sender(self, partition=None, operation=None, send_timeout=60, keep_alive=30, auto_reconnect=True, loop=None): """ Add an async sender to the client to send ~azure.eventhub.common.EventData object to an EventHub. @@ -236,13 +237,23 @@ def add_async_sender(self, partition=None, operation=None, keep_alive=30, auto_r :operation: An optional operation to be appended to the hostname in the target URL. The value must start with `/` character. :type operation: str + :param send_timeout: The timeout in seconds for an individual event to be sent from the time that it is + queued. Default value is 60 seconds. If set to 0, there will be no timeout. + :type send_timeout: int + :param keep_alive: The time interval in seconds between pinging the connection to keep it alive during + periods of inactivity. The default value is 30 seconds. If set to `None`, the connection will not + be pinged. + :type keep_alive: int + :param auto_reconnect: Whether to automatically reconnect the sender if a retryable error occurs. + Default value is `True`. + :type auto_reconnect: bool :rtype: ~azure.eventhub._async.sender_async.SenderAsync """ target = "amqps://{}{}".format(self.address.hostname, self.address.path) if operation: target = target + operation handler = AsyncSender( - self, target, partition=partition, keep_alive=keep_alive, + self, target, partition=partition, send_timeout=send_timeout, keep_alive=keep_alive, auto_reconnect=auto_reconnect, loop=loop) self.clients.append(handler) return handler diff --git a/azure/eventhub/_async/sender_async.py b/azure/eventhub/_async/sender_async.py index 7dee78e..f170d08 100644 --- a/azure/eventhub/_async/sender_async.py +++ b/azure/eventhub/_async/sender_async.py @@ -18,7 +18,7 @@ class AsyncSender(Sender): Implements the async API of a Sender. """ - def __init__(self, client, target, partition=None, keep_alive=None, auto_reconnect=True, loop=None): # pylint: disable=super-init-not-called + def __init__(self, client, target, partition=None, send_timeout=60, keep_alive=None, auto_reconnect=True, loop=None): # pylint: disable=super-init-not-called """ Instantiate an EventHub event SenderAsync handler. @@ -26,7 +26,19 @@ def __init__(self, client, target, partition=None, keep_alive=None, auto_reconne :type client: ~azure.eventhub._async.EventHubClientAsync :param target: The URI of the EventHub to send to. :type target: str - :param loop: An event loop. + :param partition: The specific partition ID to send to. Default is `None`, in which case the service + will assign to all partitions using round-robin. + :type partition: str + :param send_timeout: The timeout in seconds for an individual event to be sent from the time that it is + queued. Default value is 60 seconds. If set to 0, there will be no timeout. + :type send_timeout: int + :param keep_alive: The time interval in seconds between pinging the connection to keep it alive during + periods of inactivity. The default value is `None`, i.e. no keep alive pings. + :type keep_alive: int + :param auto_reconnect: Whether to automatically reconnect the sender if a retryable error occurs. + Default value is `True`. + :type auto_reconnect: bool + :param loop: An event loop. If not specified the default event loop will be used. """ self.loop = loop or asyncio.get_event_loop() self.client = client @@ -34,6 +46,7 @@ def __init__(self, client, target, partition=None, keep_alive=None, auto_reconne self.partition = partition self.keep_alive = keep_alive self.auto_reconnect = auto_reconnect + self.timeout = send_timeout self.retry_policy = errors.ErrorPolicy(max_retries=3, on_error=_error_handler) self.name = "EHSender-{}".format(uuid.uuid4()) self.redirected = None @@ -45,7 +58,7 @@ def __init__(self, client, target, partition=None, keep_alive=None, auto_reconne self.target, auth=self.client.get_auth(), debug=self.client.debug, - msg_timeout=Sender.TIMEOUT, + msg_timeout=self.timeout, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, @@ -69,7 +82,7 @@ async def open_async(self): self.target, auth=self.client.get_auth(), debug=self.client.debug, - msg_timeout=Sender.TIMEOUT, + msg_timeout=self.timeout, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, @@ -82,22 +95,20 @@ async def open_async(self): async def reconnect_async(self): """If the Receiver was disconnected from the service with a retryable error - attempt to reconnect.""" - # pylint: disable=protected-access - pending_states = (constants.MessageState.WaitingForSendAck, constants.MessageState.WaitingToBeSent) - unsent_events = [e for e in self._handler._pending_messages if e.state in pending_states] await self._handler.close_async() + unsent_events = self._handler.pending_messages self._handler = SendClientAsync( self.target, auth=self.client.get_auth(), debug=self.client.debug, - msg_timeout=Sender.TIMEOUT, + msg_timeout=self.timeout, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, properties=self.client.create_properties(), loop=self.loop) await self._handler.open_async() - self._handler._pending_messages = unsent_events + self._handler.queue_message(*unsent_events) await self._handler.wait_async() async def has_started(self): diff --git a/azure/eventhub/client.py b/azure/eventhub/client.py index 08f07bd..250ad7a 100644 --- a/azure/eventhub/client.py +++ b/azure/eventhub/client.py @@ -88,7 +88,7 @@ class EventHubClient(object): events to and receiving events from the Azure Event Hubs service. """ - def __init__(self, address, username=None, password=None, debug=False, http_proxy=None): + def __init__(self, address, username=None, password=None, debug=False, http_proxy=None, auth_timeout=0): """ Constructs a new EventHubClient with the given address URL. @@ -108,6 +108,9 @@ def __init__(self, address, username=None, password=None, debug=False, http_prox keys: 'proxy_hostname' (str value) and 'proxy_port' (int value). Additionally the following keys may also be present: 'username', 'password'. :type http_proxy: dict[str, Any] + :param auth_timeout: The time in seconds to wait for a token to be authorized by the service. + The default value is 60 seconds. + :type auth_timeout: int """ self.container_id = "eventhub.pysdk-" + str(uuid.uuid4())[:8] self.address = urlparse(address) @@ -124,6 +127,7 @@ def __init__(self, address, username=None, password=None, debug=False, http_prox self._auth_config = {'username': username, 'password': password} self.get_auth = functools.partial(self._create_auth) self.debug = debug + self.auth_timeout = auth_timeout self.clients = [] self.stopped = False @@ -138,6 +142,16 @@ def from_connection_string(cls, conn_str, eventhub=None, **kwargs): :type conn_str: str :param eventhub: The name of the EventHub, if the EntityName is not included in the connection string. + :param debug: Whether to output network trace logs to the logger. Default + is `False`. + :type debug: bool + :param http_proxy: HTTP proxy settings. This must be a dictionary with the following + keys: 'proxy_hostname' (str value) and 'proxy_port' (int value). + Additionally the following keys may also be present: 'username', 'password'. + :type http_proxy: dict[str, Any] + :param auth_timeout: The time in seconds to wait for a token to be authorized by the service. + The default value is 60 seconds. + :type auth_timeout: int """ address, policy, key, entity = _parse_conn_str(conn_str) entity = eventhub or entity @@ -146,6 +160,22 @@ def from_connection_string(cls, conn_str, eventhub=None, **kwargs): @classmethod def from_iothub_connection_string(cls, conn_str, **kwargs): + """ + Create an EventHubClient from an IoTHub connection string. + + :param conn_str: The connection string. + :type conn_str: str + :param debug: Whether to output network trace logs to the logger. Default + is `False`. + :type debug: bool + :param http_proxy: HTTP proxy settings. This must be a dictionary with the following + keys: 'proxy_hostname' (str value) and 'proxy_port' (int value). + Additionally the following keys may also be present: 'username', 'password'. + :type http_proxy: dict[str, Any] + :param auth_timeout: The time in seconds to wait for a token to be authorized by the service. + The default value is 60 seconds. + :type auth_timeout: int + """ address, policy, key, _ = _parse_conn_str(conn_str) hub_name = address.split('.')[0] username = "{}@sas.root.{}".format(policy, hub_name) @@ -176,7 +206,7 @@ def _create_auth(self, username=None, password=None): return authentication.SASLPlain( self.address.hostname, username, password, http_proxy=self.http_proxy) return authentication.SASTokenAuth.from_shared_access_key( - self.auth_uri, username, password, timeout=60, http_proxy=self.http_proxy) + self.auth_uri, username, password, timeout=self.auth_timeout, http_proxy=self.http_proxy) def create_properties(self): # pylint: disable=no-self-use """ @@ -362,7 +392,7 @@ def add_epoch_receiver( self.clients.append(handler) return handler - def add_sender(self, partition=None, operation=None, keep_alive=30, auto_reconnect=True): + def add_sender(self, partition=None, operation=None, send_timeout=60, keep_alive=30, auto_reconnect=True): """ Add a sender to the client to send ~azure.eventhub.common.EventData object to an EventHub. @@ -374,11 +404,20 @@ def add_sender(self, partition=None, operation=None, keep_alive=30, auto_reconne :operation: An optional operation to be appended to the hostname in the target URL. The value must start with `/` character. :type operation: str + :param send_timeout: The timeout in seconds for an individual event to be sent from the time that it is + queued. Default value is 60 seconds. If set to 0, there will be no timeout. + :type send_timeout: int + :param keep_alive: The time interval in seconds between pinging the connection to keep it alive during + periods of inactivity. The default value is 30 seconds. If set to `None`, the connection will not + be pinged. + :type keep_alive: int + :param auto_reconnect: Whether to automatically reconnect the sender if a retryable error occurs. + Default value is `True`. :rtype: ~azure.eventhub.sender.Sender """ target = "amqps://{}{}".format(self.address.hostname, self.address.path) if operation: target = target + operation - handler = Sender(self, target, partition=partition, keep_alive=keep_alive, auto_reconnect=auto_reconnect) + handler = Sender(self, target, partition=partition, send_timeout=send_timeout, keep_alive=keep_alive, auto_reconnect=auto_reconnect) self.clients.append(handler) return handler diff --git a/azure/eventhub/sender.py b/azure/eventhub/sender.py index b59ed70..c8eb544 100644 --- a/azure/eventhub/sender.py +++ b/azure/eventhub/sender.py @@ -15,9 +15,8 @@ class Sender: """ Implements a Sender. """ - TIMEOUT = 60.0 - def __init__(self, client, target, partition=None, keep_alive=None, auto_reconnect=True): + def __init__(self, client, target, partition=None, send_timeout=60, keep_alive=None, auto_reconnect=True): """ Instantiate an EventHub event Sender handler. @@ -25,10 +24,23 @@ def __init__(self, client, target, partition=None, keep_alive=None, auto_reconne :type client: ~azure.eventhub.client.EventHubClient. :param target: The URI of the EventHub to send to. :type target: str + :param partition: The specific partition ID to send to. Default is None, in which case the service + will assign to all partitions using round-robin. + :type partition: str + :param send_timeout: The timeout in seconds for an individual event to be sent from the time that it is + queued. Default value is 60 seconds. If set to 0, there will be no timeout. + :type send_timeout: int + :param keep_alive: The time interval in seconds between pinging the connection to keep it alive during + periods of inactivity. The default value is None, i.e. no keep alive pings. + :type keep_alive: int + :param auto_reconnect: Whether to automatically reconnect the sender if a retryable error occurs. + Default value is `True`. + :type auto_reconnect: bool """ self.client = client self.target = target self.partition = partition + self.timeout = send_timeout self.redirected = None self.error = None self.keep_alive = keep_alive @@ -42,7 +54,7 @@ def __init__(self, client, target, partition=None, keep_alive=None, auto_reconne self.target, auth=self.client.get_auth(), debug=self.client.debug, - msg_timeout=Sender.TIMEOUT, + msg_timeout=self.timeout, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, @@ -65,7 +77,7 @@ def open(self): self.target, auth=self.client.get_auth(), debug=self.client.debug, - msg_timeout=Sender.TIMEOUT, + msg_timeout=self.timeout, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, @@ -78,20 +90,19 @@ def reconnect(self): """If the Sender was disconnected from the service with a retryable error - attempt to reconnect.""" # pylint: disable=protected-access - pending_states = (constants.MessageState.WaitingForSendAck, constants.MessageState.WaitingToBeSent) - unsent_events = [e for e in self._handler._pending_messages if e.state in pending_states] self._handler.close() + unsent_events = self._handler.pending_messages self._handler = SendClient( self.target, auth=self.client.get_auth(), debug=self.client.debug, - msg_timeout=Sender.TIMEOUT, + msg_timeout=self.timeout, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, properties=self.client.create_properties()) self._handler.open() - self._handler._pending_messages = unsent_events + self._handler.queue_message(*unsent_events) self._handler.wait() def get_handler_state(self): From 9a335368e8d685b7ad3c6d6e6d84f01a52242e54 Mon Sep 17 00:00:00 2001 From: annatisch Date: Fri, 10 Aug 2018 11:17:54 -0700 Subject: [PATCH 31/52] Changed log formatting. Retry on reconnect --- azure/eventhub/_async/__init__.py | 10 ++-- azure/eventhub/_async/receiver_async.py | 32 ++++++++++-- azure/eventhub/_async/sender_async.py | 44 ++++++++++++++-- azure/eventhub/client.py | 10 ++-- azure/eventhub/receiver.py | 28 ++++++++-- azure/eventhub/sender.py | 28 ++++++++-- .../azure_storage_checkpoint_manager.py | 37 +++++++------ azure/eventprocessorhost/eh_partition_pump.py | 12 ++--- azure/eventprocessorhost/partition_context.py | 30 +++++------ azure/eventprocessorhost/partition_manager.py | 52 +++++++++---------- azure/eventprocessorhost/partition_pump.py | 16 +++--- 11 files changed, 201 insertions(+), 98 deletions(-) diff --git a/azure/eventhub/_async/__init__.py b/azure/eventhub/_async/__init__.py index 6868462..68a0ef6 100644 --- a/azure/eventhub/_async/__init__.py +++ b/azure/eventhub/_async/__init__.py @@ -77,7 +77,7 @@ async def _start_client_async(self, client): try: await client.open_async() except Exception as exp: # pylint: disable=broad-except - log.info("Encountered error while starting handler: {}".format(exp)) + log.info("Encountered error while starting handler: %r", exp) await client.close_async(exception=exp) log.info("Finished closing failed handler") @@ -105,17 +105,17 @@ async def run_async(self): :rtype: list[~azure.eventhub.common.EventHubError] """ - log.info("{}: Starting {} clients".format(self.container_id, len(self.clients))) + log.info("%r: Starting %r clients", self.container_id, len(self.clients)) tasks = [self._start_client_async(c) for c in self.clients] try: await asyncio.gather(*tasks) redirects = [c.redirected for c in self.clients if c.redirected] failed = [c.error for c in self.clients if c.error] if failed and len(failed) == len(self.clients): - log.warning("{}: All clients failed to start.".format(self.container_id)) + log.warning("%r: All clients failed to start.", self.container_id) raise failed[0] elif failed: - log.warning("{}: {} clients failed to start.".format(self.container_id, len(failed))) + log.warning("%r: %r clients failed to start.", self.container_id, len(failed)) elif redirects: await self._handle_redirect(redirects) except EventHubError: @@ -130,7 +130,7 @@ async def stop_async(self): """ Stop the EventHubClient and all its Sender/Receiver clients. """ - log.info("{}: Stopping {} clients".format(self.container_id, len(self.clients))) + log.info("%r: Stopping %r clients", self.container_id, len(self.clients)) self.stopped = True await self._close_clients_async() diff --git a/azure/eventhub/_async/receiver_async.py b/azure/eventhub/_async/receiver_async.py index 4cf315c..0d49231 100644 --- a/azure/eventhub/_async/receiver_async.py +++ b/azure/eventhub/_async/receiver_async.py @@ -128,9 +128,33 @@ async def reconnect_async(self): client_name=self.name, properties=self.client.create_properties(), loop=self.loop) - await self._handler.open_async() - while not await self.has_started(): - await self._handler._connection.work_async() + try: + await self._handler.open_async() + while not await self.has_started(): + await self._handler._connection.work_async() + except (errors.LinkDetach, errors.ConnectionClose) as shutdown: + if shutdown.action.retry and self.auto_reconnect: + log.info("AsyncReceiver detached. Attempting reconnect.") + await self.reconnect_async() + else: + log.info("AsyncReceiver detached. Shutting down.") + error = EventHubError(str(shutdown), shutdown) + await self.close_async(exception=error) + raise error + except errors.MessageHandlerError as shutdown: + if self.auto_reconnect: + log.info("AsyncReceiver detached. Attempting reconnect.") + await self.reconnect_async() + else: + log.info("AsyncReceiver detached. Shutting down.") + error = EventHubError(str(shutdown), shutdown) + await self.close_async(exception=error) + raise error + except Exception as e: + log.info("Unexpected error occurred (%r). Shutting down.", e) + error = EventHubError("Receiver reconnect failed: {}".format(e)) + await self.close_async(exception=error) + raise error async def has_started(self): """ @@ -224,7 +248,7 @@ async def receive(self, max_batch_size=None, timeout=None): await self.close_async(exception=error) raise error except Exception as e: - log.info("Unexpected error occurred ({}). Shutting down.".format(e)) + log.info("Unexpected error occurred (%r). Shutting down.", e) error = EventHubError("Receive failed: {}".format(e)) await self.close_async(exception=error) raise error diff --git a/azure/eventhub/_async/sender_async.py b/azure/eventhub/_async/sender_async.py index f170d08..9ef6949 100644 --- a/azure/eventhub/_async/sender_async.py +++ b/azure/eventhub/_async/sender_async.py @@ -5,6 +5,7 @@ import uuid import asyncio +import logging from uamqp import constants, errors from uamqp import SendClientAsync @@ -13,6 +14,9 @@ from azure.eventhub.sender import Sender from azure.eventhub.common import _error_handler +log = logging.getLogger(__name__) + + class AsyncSender(Sender): """ Implements the async API of a Sender. @@ -107,9 +111,33 @@ async def reconnect_async(self): client_name=self.name, properties=self.client.create_properties(), loop=self.loop) - await self._handler.open_async() - self._handler.queue_message(*unsent_events) - await self._handler.wait_async() + try: + await self._handler.open_async() + self._handler.queue_message(*unsent_events) + await self._handler.wait_async() + except (errors.LinkDetach, errors.ConnectionClose) as shutdown: + if shutdown.action.retry and self.auto_reconnect: + log.info("AsyncSender detached. Attempting reconnect.") + await self.reconnect_async() + else: + log.info("AsyncSender reconnect failed. Shutting down.") + error = EventHubError(str(shutdown), shutdown) + await self.close_async(exception=error) + raise error + except errors.MessageHandlerError as shutdown: + if self.auto_reconnect: + log.info("AsyncSender detached. Attempting reconnect.") + await self.reconnect_async() + else: + log.info("AsyncSender reconnect failed. Shutting down.") + error = EventHubError(str(shutdown), shutdown) + await self.close_async(exception=error) + raise error + except Exception as e: + log.info("Unexpected error occurred (%r). Shutting down.", e) + error = EventHubError("Sender reconnect failed: {}".format(e)) + await self.close_async(exception=error) + raise error async def has_started(self): """ @@ -178,19 +206,24 @@ async def send(self, event_data): raise Sender._error(self._outcome, self._condition) except (errors.LinkDetach, errors.ConnectionClose) as shutdown: if shutdown.action.retry and self.auto_reconnect: + log.info("AsyncSender detached. Attempting reconnect.") await self.reconnect_async() else: + log.info("AsyncSender detached. Shutting down.") error = EventHubError(str(shutdown), shutdown) await self.close_async(exception=error) raise error except errors.MessageHandlerError as shutdown: if self.auto_reconnect: + log.info("AsyncSender detached. Attempting reconnect.") await self.reconnect_async() else: + log.info("AsyncSender detached. Shutting down.") error = EventHubError(str(shutdown), shutdown) await self.close_async(exception=error) raise error except Exception as e: + log.info("Unexpected error occurred (%r). Shutting down.", e) error = EventHubError("Send failed: {}".format(e)) await self.close_async(exception=error) raise error @@ -207,17 +240,22 @@ async def wait_async(self): await self._handler.wait_async() except (errors.LinkDetach, errors.ConnectionClose) as shutdown: if shutdown.action.retry and self.auto_reconnect: + log.info("AsyncSender detached. Attempting reconnect.") await self.reconnect_async() else: + log.info("AsyncSender detached. Shutting down.") error = EventHubError(str(shutdown), shutdown) await self.close_async(exception=error) raise error except errors.MessageHandlerError as shutdown: if self.auto_reconnect: + log.info("AsyncSender detached. Attempting reconnect.") await self.reconnect_async() else: + log.info("AsyncSender detached. Shutting down.") error = EventHubError(str(shutdown), shutdown) await self.close_async(exception=error) raise error except Exception as e: + log.info("Unexpected error occurred (%r).", e) raise EventHubError("Send failed: {}".format(e)) diff --git a/azure/eventhub/client.py b/azure/eventhub/client.py index 250ad7a..7f252e5 100644 --- a/azure/eventhub/client.py +++ b/azure/eventhub/client.py @@ -131,7 +131,7 @@ def __init__(self, address, username=None, password=None, debug=False, http_prox self.clients = [] self.stopped = False - log.info("{}: Created the Event Hub client".format(self.container_id)) + log.info("%r: Created the Event Hub client", self.container_id) @classmethod def from_connection_string(cls, conn_str, eventhub=None, **kwargs): @@ -265,16 +265,16 @@ def run(self): :rtype: list[~azure.eventhub.common.EventHubError] """ - log.info("{}: Starting {} clients".format(self.container_id, len(self.clients))) + log.info("%r: Starting %r clients", self.container_id, len(self.clients)) try: self._start_clients() redirects = [c.redirected for c in self.clients if c.redirected] failed = [c.error for c in self.clients if c.error] if failed and len(failed) == len(self.clients): - log.warning("{}: All clients failed to start.".format(self.container_id)) + log.warning("%r: All clients failed to start.", self.container_id) raise failed[0] elif failed: - log.warning("{}: {} clients failed to start.".format(self.container_id, len(failed))) + log.warning("%r: %r clients failed to start.", self.container_id, len(failed)) elif redirects: self._handle_redirect(redirects) except EventHubError: @@ -289,7 +289,7 @@ def stop(self): """ Stop the EventHubClient and all its Sender/Receiver clients. """ - log.info("{}: Stopping {} clients".format(self.container_id, len(self.clients))) + log.info("%r: Stopping %r clients", self.container_id, len(self.clients)) self.stopped = True self._close_clients() diff --git a/azure/eventhub/receiver.py b/azure/eventhub/receiver.py index 90af41e..0b7b8a9 100644 --- a/azure/eventhub/receiver.py +++ b/azure/eventhub/receiver.py @@ -4,12 +4,15 @@ # -------------------------------------------------------------------------------------------- import uuid +import logging from uamqp import types, errors from uamqp import ReceiveClient, Source from azure.eventhub.common import EventHubError, EventData, _error_handler +log = logging.getLogger(__name__) + class Receiver: """ @@ -117,9 +120,28 @@ def reconnect(self): keep_alive_interval=self.keep_alive, client_name=self.name, properties=self.client.create_properties()) - self._handler.open() - while not self.has_started(): - self._handler._connection.work() + try: + self._handler.open() + while not self.has_started(): + self._handler._connection.work() + except (errors.LinkDetach, errors.ConnectionClose) as shutdown: + if shutdown.action.retry and self.auto_reconnect: + self.reconnect() + else: + error = EventHubError(str(shutdown), shutdown) + self.close(exception=error) + raise error + except errors.MessageHandlerError as shutdown: + if self.auto_reconnect: + self.reconnect() + else: + error = EventHubError(str(shutdown), shutdown) + self.close(exception=error) + raise error + except Exception as e: + error = EventHubError("Receiver reconnect failed: {}".format(e)) + self.close(exception=error) + raise error def get_handler_state(self): """ diff --git a/azure/eventhub/sender.py b/azure/eventhub/sender.py index c8eb544..e0ed738 100644 --- a/azure/eventhub/sender.py +++ b/azure/eventhub/sender.py @@ -4,12 +4,15 @@ # -------------------------------------------------------------------------------------------- import uuid +import logging from uamqp import constants, errors from uamqp import SendClient from azure.eventhub.common import EventHubError, _error_handler +log = logging.getLogger(__name__) + class Sender: """ @@ -101,9 +104,28 @@ def reconnect(self): keep_alive_interval=self.keep_alive, client_name=self.name, properties=self.client.create_properties()) - self._handler.open() - self._handler.queue_message(*unsent_events) - self._handler.wait() + try: + self._handler.open() + self._handler.queue_message(*unsent_events) + self._handler.wait() + except (errors.LinkDetach, errors.ConnectionClose) as shutdown: + if shutdown.action.retry and self.auto_reconnect: + self.reconnect() + else: + error = EventHubError(str(shutdown), shutdown) + self.close(exception=error) + raise error + except errors.MessageHandlerError as shutdown: + if self.auto_reconnect: + self.reconnect() + else: + error = EventHubError(str(shutdown), shutdown) + self.close(exception=error) + raise error + except Exception as e: + error = EventHubError("Sender Reconnect failed: {}".format(e)) + self.close(exception=error) + raise error def get_handler_state(self): """ diff --git a/azure/eventprocessorhost/azure_storage_checkpoint_manager.py b/azure/eventprocessorhost/azure_storage_checkpoint_manager.py index 001eafa..25351f2 100644 --- a/azure/eventprocessorhost/azure_storage_checkpoint_manager.py +++ b/azure/eventprocessorhost/azure_storage_checkpoint_manager.py @@ -160,7 +160,7 @@ async def create_lease_store_if_not_exists_async(self): self.lease_container_name)) except Exception as err: # pylint: disable=broad-except - _logger.error(repr(err)) + _logger.error("%r", err) raise err return True @@ -206,12 +206,12 @@ async def state(): partition_id)) return res.properties.lease.state except Exception as err: # pylint: disable=broad-except - _logger.error("Failed to get lease state {} {}".format(err, partition_id)) + _logger.error("Failed to get lease state %r %r", err, partition_id) lease.state = state return lease except Exception as err: # pylint: disable=broad-except - _logger.error("Failed to get lease {} {}".format(err, partition_id)) + _logger.error("Failed to get lease %r %r", err, partition_id) async def get_all_leases(self): """ @@ -242,10 +242,10 @@ async def create_lease_if_not_exists_async(self, partition_id): return_lease = AzureBlobLease() return_lease.partition_id = partition_id json_lease = json.dumps(return_lease.serializable()) - _logger.info("Creating Lease {} {} {}".format( + _logger.info("Creating Lease %r %r %r", self.lease_container_name, partition_id, - json_lease)) + json_lease) await self.host.loop.run_in_executor( self.executor, functools.partial( @@ -257,7 +257,7 @@ async def create_lease_if_not_exists_async(self, partition_id): try: return_lease = await self.get_lease_async(partition_id) except Exception as err: # pylint: disable=broad-except - _logger.error("Failed to create lease {!r}".format(err)) + _logger.error("Failed to create lease %r", err) raise err return return_lease @@ -308,7 +308,7 @@ async def acquire_lease_async(self, lease): # than it should, rebalancing will take care of that quickly enough. retval = False else: - _logger.info("ChangingLease {} {}".format(self.host.guid, lease.partition_id)) + _logger.info("ChangingLease %r %r", self.host.guid, lease.partition_id) await self.host.loop.run_in_executor( self.executor, functools.partial( @@ -319,7 +319,7 @@ async def acquire_lease_async(self, lease): new_lease_id)) lease.token = new_lease_id else: - _logger.info("AcquiringLease {} {}".format(self.host.guid, lease.partition_id)) + _logger.info("AcquiringLease %r %r", self.host.guid, lease.partition_id) lease.token = await self.host.loop.run_in_executor( self.executor, functools.partial( @@ -333,8 +333,7 @@ async def acquire_lease_async(self, lease): # check if this solves the issue retval = await self.update_lease_async(lease) except Exception as err: # pylint: disable=broad-except - _logger.error("Failed to acquire lease {!r} {} {}".format( - err, partition_id, lease.token)) + _logger.error("Failed to acquire lease %r %r %r", err, partition_id, lease.token) return False return retval @@ -361,10 +360,10 @@ async def renew_lease_async(self, lease): timeout=self.lease_duration)) except Exception as err: # pylint: disable=broad-except if "LeaseIdMismatchWithLeaseOperation" in str(err): - _logger.info("LeaseLost on partition {}".format(lease.partition_id)) + _logger.info("LeaseLost on partition %r", lease.partition_id) else: - _logger.error("Failed to renew lease on partition {} with token {} {!r}".format( - lease.partition_id, lease.token, err)) + _logger.error("Failed to renew lease on partition %r with token %r %r", + lease.partition_id, lease.token, err) return False return True @@ -380,7 +379,7 @@ async def release_lease_async(self, lease): """ lease_id = None try: - _logger.info("Releasing lease {} {}".format(self.host.guid, lease.partition_id)) + _logger.info("Releasing lease %r %r", self.host.guid, lease.partition_id) lease_id = lease.token released_copy = AzureBlobLease() released_copy.with_lease(lease) @@ -403,8 +402,8 @@ async def release_lease_async(self, lease): lease.partition_id, lease_id)) except Exception as err: # pylint: disable=broad-except - _logger.error("Failed to release lease {} {} {}".format( - err, lease.partition_id, lease_id)) + _logger.error("Failed to release lease %r %r %r", + err, lease.partition_id, lease_id) return False return True @@ -426,7 +425,7 @@ async def update_lease_async(self, lease): if not lease.token: return False - _logger.debug("Updating lease {} {}".format(self.host.guid, lease.partition_id)) + _logger.debug("Updating lease %r %r", self.host.guid, lease.partition_id) # First, renew the lease to make sure the update will go through. if await self.renew_lease_async(lease): @@ -441,8 +440,8 @@ async def update_lease_async(self, lease): lease_id=lease.token)) except Exception as err: # pylint: disable=broad-except - _logger.error("Failed to update lease {} {} {}".format( - self.host.guid, lease.partition_id, err)) + _logger.error("Failed to update lease %r %r %r", + self.host.guid, lease.partition_id, err) raise err else: return False diff --git a/azure/eventprocessorhost/eh_partition_pump.py b/azure/eventprocessorhost/eh_partition_pump.py index 368c2bb..5832b49 100644 --- a/azure/eventprocessorhost/eh_partition_pump.py +++ b/azure/eventprocessorhost/eh_partition_pump.py @@ -36,8 +36,8 @@ async def on_open_async(self): _opened_ok = True except Exception as err: # pylint: disable=broad-except _logger.warning( - "{},{} PartitionPumpWarning: Failure creating client or receiver, " - "retrying: {!r}".format(self.host.guid, self.partition_context.partition_id, err)) + "%r,%r PartitionPumpWarning: Failure creating client or receiver, " + "retrying: %r", self.host.guid, self.partition_context.partition_id, err) last_exception = err _retry_count += 1 @@ -102,7 +102,7 @@ async def on_closing_async(self, reason): except TypeError: _logger.debug("No partition pump running.") except Exception as err: # pylint: disable=broad-except - _logger.info("Error on closing partition pump: {!r}".format(err)) + _logger.info("Error on closing partition pump: %r", err) await self.clean_up_clients_async() @@ -128,13 +128,13 @@ async def run(self): max_batch_size=self.max_batch_size, timeout=self.recieve_timeout) except Exception as e: # pylint: disable=broad-except - _logger.info("Error raised while attempting to receive messages: {}".format(e)) + _logger.info("Error raised while attempting to receive messages: %r", e) await self.process_error_async(e) else: if not msgs: - _logger.info("No events received, queue size {}, release {}".format( + _logger.info("No events received, queue size %r, release %r", self.eh_partition_pump.partition_receive_handler.queue_size, - self.eh_partition_pump.host.eph_options.release_pump_on_timeout)) + self.eh_partition_pump.host.eph_options.release_pump_on_timeout) if self.eh_partition_pump.host.eph_options.release_pump_on_timeout: await self.process_error_async(TimeoutError("No events received")) else: diff --git a/azure/eventprocessorhost/partition_context.py b/azure/eventprocessorhost/partition_context.py index 33cc566..fb6926e 100644 --- a/azure/eventprocessorhost/partition_context.py +++ b/azure/eventprocessorhost/partition_context.py @@ -43,8 +43,8 @@ async def get_initial_offset_async(self): # throws InterruptedException, Executi :rtype: str """ - _logger.info("Calling user-provided initial offset provider {} {}".format( - self.host.guid, self.partition_id)) + _logger.info("Calling user-provided initial offset provider %r %r", + self.host.guid, self.partition_id) starting_checkpoint = await self.host.storage_manager.get_checkpoint_async(self.partition_id) if not starting_checkpoint: # No checkpoint was ever stored. Use the initialOffsetProvider instead @@ -55,8 +55,8 @@ async def get_initial_offset_async(self): # throws InterruptedException, Executi self.offset = starting_checkpoint.offset self.sequence_number = starting_checkpoint.sequence_number - _logger.info("{} {} Initial offset/sequenceNumber provided {}/{}".format( - self.host.guid, self.partition_id, self.offset, self.sequence_number)) + _logger.info("%r %r Initial offset/sequenceNumber provided %r/%r", + self.host.guid, self.partition_id, self.offset, self.sequence_number) return self.offset async def checkpoint_async(self): @@ -106,34 +106,34 @@ async def persist_checkpoint_async(self, checkpoint): :param checkpoint: The checkpoint to persist. :type checkpoint: ~azure.eventprocessorhost.checkpoint.Checkpoint """ - _logger.debug("PartitionPumpCheckpointStart {} {} {} {}".format( - self.host.guid, checkpoint.partition_id, checkpoint.offset, checkpoint.sequence_number)) + _logger.debug("PartitionPumpCheckpointStart %r %r %r %r", + self.host.guid, checkpoint.partition_id, checkpoint.offset, checkpoint.sequence_number) try: in_store_checkpoint = await self.host.storage_manager.get_checkpoint_async(checkpoint.partition_id) if not in_store_checkpoint or checkpoint.sequence_number >= in_store_checkpoint.sequence_number: if not in_store_checkpoint: - _logger.info("persisting checkpoint {}".format(checkpoint.__dict__)) + _logger.info("persisting checkpoint %r", checkpoint.__dict__) await self.host.storage_manager.create_checkpoint_if_not_exists_async(checkpoint.partition_id) if not await self.host.storage_manager.update_checkpoint_async(self.lease, checkpoint): - _logger.error("Failed to persist checkpoint for partition: {}".format(self.partition_id)) + _logger.error("Failed to persist checkpoint for partition: %r", self.partition_id) raise Exception("failed to persist checkpoint") self.lease.offset = checkpoint.offset self.lease.sequence_number = checkpoint.sequence_number else: _logger.error( - "Ignoring out of date checkpoint with offset {}/sequence number {} because " - "current persisted checkpoint has higher offset {}/sequence number {}".format( + "Ignoring out of date checkpoint with offset %r/sequence number %r because " + "current persisted checkpoint has higher offset %r/sequence number %r", checkpoint.offset, checkpoint.sequence_number, in_store_checkpoint.offset, - in_store_checkpoint.sequence_number)) + in_store_checkpoint.sequence_number) raise Exception("offset/sequenceNumber invalid") except Exception as err: - _logger.error("PartitionPumpCheckpointError {} {} {!r}".format( - self.host.guid, checkpoint.partition_id, err)) + _logger.error("PartitionPumpCheckpointError %r %r %r", + self.host.guid, checkpoint.partition_id, err) raise finally: - _logger.debug("PartitionPumpCheckpointStop {} {}".format( - self.host.guid, checkpoint.partition_id)) + _logger.debug("PartitionPumpCheckpointStop %r %r", + self.host.guid, checkpoint.partition_id) diff --git a/azure/eventprocessorhost/partition_manager.py b/azure/eventprocessorhost/partition_manager.py index c9c927c..5778ce3 100644 --- a/azure/eventprocessorhost/partition_manager.py +++ b/azure/eventprocessorhost/partition_manager.py @@ -57,7 +57,7 @@ async def start_async(self): raise Exception("A PartitionManager cannot be started multiple times.") partition_count = await self.initialize_stores_async() - _logger.info("{} PartitionCount: {}".format(self.host.guid, partition_count)) + _logger.info("%r PartitionCount: %r", self.host.guid, partition_count) self.run_task = asyncio.ensure_future(self.run_async()) async def stop_async(self): @@ -75,10 +75,10 @@ async def run_async(self): try: await self.run_loop_async() except Exception as err: # pylint: disable=broad-except - _logger.error("Run loop failed {!r}".format(err)) + _logger.error("Run loop failed %r", err) try: - _logger.info("Shutting down all pumps {}".format(self.host.guid)) + _logger.info("Shutting down all pumps %r", self.host.guid) await self.remove_all_pumps_async("Shutdown") except Exception as err: # pylint: disable=broad-except raise Exception("Failed to remove all pumps {!r}".format(err)) @@ -128,7 +128,7 @@ async def retry_async(self, func, partition_id, retry_message, await func(partition_id) created_okay = True except Exception as err: # pylint: disable=broad-except - _logger.error("{} {} {} {!r}".format(retry_message, host_id, partition_id, err)) + _logger.error("%r %r %r %r", retry_message, host_id, partition_id, err) retry_count += 1 if not created_okay: raise Exception(host_id, final_failure_message) @@ -172,28 +172,28 @@ async def run_loop_async(self): leases_owned_by_others, our_lease_count) if steal_this_lease: try: - _logger.info("Lease to steal {}".format(steal_this_lease.serializable())) + _logger.info("Lease to steal %r", steal_this_lease.serializable()) if await lease_manager.acquire_lease_async(steal_this_lease): - _logger.info("Stole lease sucessfully {} {}".format( - self.host.guid, steal_this_lease.partition_id)) + _logger.info("Stole lease sucessfully %r %r", + self.host.guid, steal_this_lease.partition_id) else: - _logger.info("Failed to steal lease for partition {} {}".format( - self.host.guid, steal_this_lease.partition_id)) + _logger.info("Failed to steal lease for partition %r %r", + self.host.guid, steal_this_lease.partition_id) except Exception as err: # pylint: disable=broad-except - _logger.error("Failed to steal lease {!r}".format(err)) + _logger.error("Failed to steal lease %r", err) for partition_id in all_leases: try: updated_lease = all_leases[partition_id] if updated_lease.owner == self.host.host_name: - _logger.debug("Attempting to renew lease {} {}".format( - self.host.guid, partition_id)) + _logger.debug("Attempting to renew lease %r %r", + self.host.guid, partition_id) await self.check_and_add_pump_async(partition_id, updated_lease) else: _logger.debug("Removing pump due to lost lease.") await self.remove_pump_async(partition_id, "LeaseLost") except Exception as err: # pylint: disable=broad-except - _logger.error("Failed to update lease {!r}".format(err)) + _logger.error("Failed to update lease %r", err) await asyncio.sleep(lease_manager.lease_renew_interval) async def check_and_add_pump_async(self, partition_id, lease): @@ -217,7 +217,7 @@ async def check_and_add_pump_async(self, partition_id, lease): # when the lease changes then the pump will error and shut down captured_pump.set_lease(lease) else: - _logger.info("Starting pump {} {}".format(self.host.guid, partition_id)) + _logger.info("Starting pump %r %r", self.host.guid, partition_id) await self.create_new_pump_async(partition_id, lease) async def create_new_pump_async(self, partition_id, lease): @@ -234,7 +234,7 @@ async def create_new_pump_async(self, partition_id, lease): # Do the put after start, if the start fails then put doesn't happen loop.create_task(partition_pump.open_async()) self.partition_pumps[partition_id] = partition_pump - _logger.info("Created new partition pump {} {}".format(self.host.guid, partition_id)) + _logger.info("Created new partition pump %r %r", self.host.guid, partition_id) async def remove_pump_async(self, partition_id, reason): """ @@ -251,14 +251,14 @@ async def remove_pump_async(self, partition_id, reason): await captured_pump.close_async(reason) # else, pump is already closing/closed, don't need to try to shut it down again del self.partition_pumps[partition_id] # remove pump - _logger.debug("Removed pump {} {}".format(self.host.guid, partition_id)) - _logger.debug("{} pumps still running".format(len(self.partition_pumps))) + _logger.debug("Removed pump %r %r", self.host.guid, partition_id) + _logger.debug("%r pumps still running", len(self.partition_pumps)) else: # PartitionManager main loop tries to remove pump for every partition that the # host does not own, just to be sure. Not finding a pump for a partition is normal # and expected most of the time. - _logger.debug("No pump found to remove for this partition {} {}".format( - self.host.guid, partition_id)) + _logger.debug("No pump found to remove for this partition %r %r", + self.host.guid, partition_id) async def remove_all_pumps_async(self, reason): """ @@ -332,8 +332,8 @@ async def attempt_renew_lease_async(self, lease_task, owned_by_others_q, lease_m try: possible_lease = await lease_task if await possible_lease.is_expired(): - logging.info("Trying to aquire lease {} {}".format( - self.host.guid, possible_lease.partition_id)) + _logger.info("Trying to aquire lease %r %r", + self.host.guid, possible_lease.partition_id) if await lease_manager.acquire_lease_async(possible_lease): owned_by_others_q.put((False, possible_lease)) else: @@ -341,20 +341,20 @@ async def attempt_renew_lease_async(self, lease_task, owned_by_others_q, lease_m elif possible_lease.owner == self.host.host_name: try: - _logger.debug("Trying to renew lease {} {}".format( - self.host.guid, possible_lease.partition_id)) + _logger.debug("Trying to renew lease %r %r", + self.host.guid, possible_lease.partition_id) if await lease_manager.renew_lease_async(possible_lease): owned_by_others_q.put((False, possible_lease)) else: owned_by_others_q.put((True, possible_lease)) except Exception as err: # pylint: disable=broad-except # Update to 'Lease Lost' exception. - _logger.error("Lease lost exception {!r} {} {}".format( - err, self.host.guid, possible_lease.partition_id)) + _logger.error("Lease lost exception %r %r %r", + err, self.host.guid, possible_lease.partition_id) owned_by_others_q.put((True, possible_lease)) else: owned_by_others_q.put((True, possible_lease)) except Exception as err: # pylint: disable=broad-except _logger.error( - "Failure during getting/acquiring/renewing lease, skipping {!r}".format(err)) + "Failure during getting/acquiring/renewing lease, skipping %r", err) diff --git a/azure/eventprocessorhost/partition_pump.py b/azure/eventprocessorhost/partition_pump.py index 769e2ce..57c7ae7 100644 --- a/azure/eventprocessorhost/partition_pump.py +++ b/azure/eventprocessorhost/partition_pump.py @@ -37,7 +37,7 @@ def set_pump_status(self, status): Updates pump status and logs update to console. """ self.pump_status = status - _logger.info("{} partition {}".format(status, self.lease.partition_id)) + _logger.info("%r partition %r", status, self.lease.partition_id) def set_lease(self, new_lease): """ @@ -99,15 +99,14 @@ async def close_async(self, reason): try: await self.on_closing_async(reason) if self.processor: - _logger.info("PartitionPumpInvokeProcessorCloseStart {} {} {}".format( - self.host.guid, self.partition_context.partition_id, reason)) + _logger.info("PartitionPumpInvokeProcessorCloseStart %r %r %r", + self.host.guid, self.partition_context.partition_id, reason) await self.processor.close_async(self.partition_context, reason) - _logger.info("PartitionPumpInvokeProcessorCloseStart {} {}".format( - self.host.guid, self.partition_context.partition_id)) + _logger.info("PartitionPumpInvokeProcessorCloseStart %r %r", + self.host.guid, self.partition_context.partition_id) except Exception as err: # pylint: disable=broad-except await self.process_error_async(err) - _logger.error("{} {} {!r}".format( - self.host.guid, self.partition_context.partition_id, err)) + _logger.error("%r %r %r", self.host.guid, self.partition_context.partition_id, err) raise err if reason == "LeaseLost": @@ -115,8 +114,7 @@ async def close_async(self, reason): _logger.info("Lease Lost releasing ownership") await self.host.storage_manager.release_lease_async(self.partition_context.lease) except Exception as err: # pylint: disable=broad-except - _logger.error("{} {} {!r}".format( - self.host.guid, self.partition_context.partition_id, err)) + _logger.error("%r %r %r", self.host.guid, self.partition_context.partition_id, err) raise err self.set_pump_status("Closed") From 5692cb0dabe0a5cad315320009d4903e5dcff68e Mon Sep 17 00:00:00 2001 From: annatisch Date: Mon, 20 Aug 2018 11:26:43 -0700 Subject: [PATCH 32/52] Pylint fixes --- azure/eventhub/_async/__init__.py | 4 +++- azure/eventhub/_async/sender_async.py | 4 +++- azure/eventhub/client.py | 6 ++++-- .../azure_storage_checkpoint_manager.py | 12 +++++------ azure/eventprocessorhost/eh_partition_pump.py | 4 ++-- azure/eventprocessorhost/partition_context.py | 20 +++++++++---------- azure/eventprocessorhost/partition_manager.py | 14 ++++++------- azure/eventprocessorhost/partition_pump.py | 4 ++-- 8 files changed, 37 insertions(+), 31 deletions(-) diff --git a/azure/eventhub/_async/__init__.py b/azure/eventhub/_async/__init__.py index 68a0ef6..5b102b5 100644 --- a/azure/eventhub/_async/__init__.py +++ b/azure/eventhub/_async/__init__.py @@ -225,7 +225,9 @@ def add_async_epoch_receiver( self.clients.append(handler) return handler - def add_async_sender(self, partition=None, operation=None, send_timeout=60, keep_alive=30, auto_reconnect=True, loop=None): + def add_async_sender( + self, partition=None, operation=None, send_timeout=60, + keep_alive=30, auto_reconnect=True, loop=None): """ Add an async sender to the client to send ~azure.eventhub.common.EventData object to an EventHub. diff --git a/azure/eventhub/_async/sender_async.py b/azure/eventhub/_async/sender_async.py index 9ef6949..bd72dd5 100644 --- a/azure/eventhub/_async/sender_async.py +++ b/azure/eventhub/_async/sender_async.py @@ -22,7 +22,9 @@ class AsyncSender(Sender): Implements the async API of a Sender. """ - def __init__(self, client, target, partition=None, send_timeout=60, keep_alive=None, auto_reconnect=True, loop=None): # pylint: disable=super-init-not-called + def __init__( # pylint: disable=super-init-not-called + self, client, target, partition=None, send_timeout=60, + keep_alive=None, auto_reconnect=True, loop=None): """ Instantiate an EventHub event SenderAsync handler. diff --git a/azure/eventhub/client.py b/azure/eventhub/client.py index 7f252e5..3a61c60 100644 --- a/azure/eventhub/client.py +++ b/azure/eventhub/client.py @@ -88,7 +88,7 @@ class EventHubClient(object): events to and receiving events from the Azure Event Hubs service. """ - def __init__(self, address, username=None, password=None, debug=False, http_proxy=None, auth_timeout=0): + def __init__(self, address, username=None, password=None, debug=False, http_proxy=None, auth_timeout=60): """ Constructs a new EventHubClient with the given address URL. @@ -418,6 +418,8 @@ def add_sender(self, partition=None, operation=None, send_timeout=60, keep_alive target = "amqps://{}{}".format(self.address.hostname, self.address.path) if operation: target = target + operation - handler = Sender(self, target, partition=partition, send_timeout=send_timeout, keep_alive=keep_alive, auto_reconnect=auto_reconnect) + handler = Sender( + self, target, partition=partition, send_timeout=send_timeout, + keep_alive=keep_alive, auto_reconnect=auto_reconnect) self.clients.append(handler) return handler diff --git a/azure/eventprocessorhost/azure_storage_checkpoint_manager.py b/azure/eventprocessorhost/azure_storage_checkpoint_manager.py index 25351f2..8ac3abe 100644 --- a/azure/eventprocessorhost/azure_storage_checkpoint_manager.py +++ b/azure/eventprocessorhost/azure_storage_checkpoint_manager.py @@ -243,9 +243,9 @@ async def create_lease_if_not_exists_async(self, partition_id): return_lease.partition_id = partition_id json_lease = json.dumps(return_lease.serializable()) _logger.info("Creating Lease %r %r %r", - self.lease_container_name, - partition_id, - json_lease) + self.lease_container_name, + partition_id, + json_lease) await self.host.loop.run_in_executor( self.executor, functools.partial( @@ -363,7 +363,7 @@ async def renew_lease_async(self, lease): _logger.info("LeaseLost on partition %r", lease.partition_id) else: _logger.error("Failed to renew lease on partition %r with token %r %r", - lease.partition_id, lease.token, err) + lease.partition_id, lease.token, err) return False return True @@ -403,7 +403,7 @@ async def release_lease_async(self, lease): lease_id)) except Exception as err: # pylint: disable=broad-except _logger.error("Failed to release lease %r %r %r", - err, lease.partition_id, lease_id) + err, lease.partition_id, lease_id) return False return True @@ -441,7 +441,7 @@ async def update_lease_async(self, lease): except Exception as err: # pylint: disable=broad-except _logger.error("Failed to update lease %r %r %r", - self.host.guid, lease.partition_id, err) + self.host.guid, lease.partition_id, err) raise err else: return False diff --git a/azure/eventprocessorhost/eh_partition_pump.py b/azure/eventprocessorhost/eh_partition_pump.py index 5832b49..995440a 100644 --- a/azure/eventprocessorhost/eh_partition_pump.py +++ b/azure/eventprocessorhost/eh_partition_pump.py @@ -133,8 +133,8 @@ async def run(self): else: if not msgs: _logger.info("No events received, queue size %r, release %r", - self.eh_partition_pump.partition_receive_handler.queue_size, - self.eh_partition_pump.host.eph_options.release_pump_on_timeout) + self.eh_partition_pump.partition_receive_handler.queue_size, + self.eh_partition_pump.host.eph_options.release_pump_on_timeout) if self.eh_partition_pump.host.eph_options.release_pump_on_timeout: await self.process_error_async(TimeoutError("No events received")) else: diff --git a/azure/eventprocessorhost/partition_context.py b/azure/eventprocessorhost/partition_context.py index fb6926e..510fdd6 100644 --- a/azure/eventprocessorhost/partition_context.py +++ b/azure/eventprocessorhost/partition_context.py @@ -44,7 +44,7 @@ async def get_initial_offset_async(self): # throws InterruptedException, Executi :rtype: str """ _logger.info("Calling user-provided initial offset provider %r %r", - self.host.guid, self.partition_id) + self.host.guid, self.partition_id) starting_checkpoint = await self.host.storage_manager.get_checkpoint_async(self.partition_id) if not starting_checkpoint: # No checkpoint was ever stored. Use the initialOffsetProvider instead @@ -56,7 +56,7 @@ async def get_initial_offset_async(self): # throws InterruptedException, Executi self.sequence_number = starting_checkpoint.sequence_number _logger.info("%r %r Initial offset/sequenceNumber provided %r/%r", - self.host.guid, self.partition_id, self.offset, self.sequence_number) + self.host.guid, self.partition_id, self.offset, self.sequence_number) return self.offset async def checkpoint_async(self): @@ -107,7 +107,7 @@ async def persist_checkpoint_async(self, checkpoint): :type checkpoint: ~azure.eventprocessorhost.checkpoint.Checkpoint """ _logger.debug("PartitionPumpCheckpointStart %r %r %r %r", - self.host.guid, checkpoint.partition_id, checkpoint.offset, checkpoint.sequence_number) + self.host.guid, checkpoint.partition_id, checkpoint.offset, checkpoint.sequence_number) try: in_store_checkpoint = await self.host.storage_manager.get_checkpoint_async(checkpoint.partition_id) if not in_store_checkpoint or checkpoint.sequence_number >= in_store_checkpoint.sequence_number: @@ -122,18 +122,18 @@ async def persist_checkpoint_async(self, checkpoint): self.lease.sequence_number = checkpoint.sequence_number else: _logger.error( - "Ignoring out of date checkpoint with offset %r/sequence number %r because " + "Ignoring out of date checkpoint with offset %r/sequence number %r because " + "current persisted checkpoint has higher offset %r/sequence number %r", - checkpoint.offset, - checkpoint.sequence_number, - in_store_checkpoint.offset, - in_store_checkpoint.sequence_number) + checkpoint.offset, + checkpoint.sequence_number, + in_store_checkpoint.offset, + in_store_checkpoint.sequence_number) raise Exception("offset/sequenceNumber invalid") except Exception as err: _logger.error("PartitionPumpCheckpointError %r %r %r", - self.host.guid, checkpoint.partition_id, err) + self.host.guid, checkpoint.partition_id, err) raise finally: _logger.debug("PartitionPumpCheckpointStop %r %r", - self.host.guid, checkpoint.partition_id) + self.host.guid, checkpoint.partition_id) diff --git a/azure/eventprocessorhost/partition_manager.py b/azure/eventprocessorhost/partition_manager.py index 5778ce3..41aaded 100644 --- a/azure/eventprocessorhost/partition_manager.py +++ b/azure/eventprocessorhost/partition_manager.py @@ -175,10 +175,10 @@ async def run_loop_async(self): _logger.info("Lease to steal %r", steal_this_lease.serializable()) if await lease_manager.acquire_lease_async(steal_this_lease): _logger.info("Stole lease sucessfully %r %r", - self.host.guid, steal_this_lease.partition_id) + self.host.guid, steal_this_lease.partition_id) else: _logger.info("Failed to steal lease for partition %r %r", - self.host.guid, steal_this_lease.partition_id) + self.host.guid, steal_this_lease.partition_id) except Exception as err: # pylint: disable=broad-except _logger.error("Failed to steal lease %r", err) @@ -187,7 +187,7 @@ async def run_loop_async(self): updated_lease = all_leases[partition_id] if updated_lease.owner == self.host.host_name: _logger.debug("Attempting to renew lease %r %r", - self.host.guid, partition_id) + self.host.guid, partition_id) await self.check_and_add_pump_async(partition_id, updated_lease) else: _logger.debug("Removing pump due to lost lease.") @@ -258,7 +258,7 @@ async def remove_pump_async(self, partition_id, reason): # host does not own, just to be sure. Not finding a pump for a partition is normal # and expected most of the time. _logger.debug("No pump found to remove for this partition %r %r", - self.host.guid, partition_id) + self.host.guid, partition_id) async def remove_all_pumps_async(self, reason): """ @@ -333,7 +333,7 @@ async def attempt_renew_lease_async(self, lease_task, owned_by_others_q, lease_m possible_lease = await lease_task if await possible_lease.is_expired(): _logger.info("Trying to aquire lease %r %r", - self.host.guid, possible_lease.partition_id) + self.host.guid, possible_lease.partition_id) if await lease_manager.acquire_lease_async(possible_lease): owned_by_others_q.put((False, possible_lease)) else: @@ -342,7 +342,7 @@ async def attempt_renew_lease_async(self, lease_task, owned_by_others_q, lease_m elif possible_lease.owner == self.host.host_name: try: _logger.debug("Trying to renew lease %r %r", - self.host.guid, possible_lease.partition_id) + self.host.guid, possible_lease.partition_id) if await lease_manager.renew_lease_async(possible_lease): owned_by_others_q.put((False, possible_lease)) else: @@ -350,7 +350,7 @@ async def attempt_renew_lease_async(self, lease_task, owned_by_others_q, lease_m except Exception as err: # pylint: disable=broad-except # Update to 'Lease Lost' exception. _logger.error("Lease lost exception %r %r %r", - err, self.host.guid, possible_lease.partition_id) + err, self.host.guid, possible_lease.partition_id) owned_by_others_q.put((True, possible_lease)) else: owned_by_others_q.put((True, possible_lease)) diff --git a/azure/eventprocessorhost/partition_pump.py b/azure/eventprocessorhost/partition_pump.py index 57c7ae7..be8be04 100644 --- a/azure/eventprocessorhost/partition_pump.py +++ b/azure/eventprocessorhost/partition_pump.py @@ -100,10 +100,10 @@ async def close_async(self, reason): await self.on_closing_async(reason) if self.processor: _logger.info("PartitionPumpInvokeProcessorCloseStart %r %r %r", - self.host.guid, self.partition_context.partition_id, reason) + self.host.guid, self.partition_context.partition_id, reason) await self.processor.close_async(self.partition_context, reason) _logger.info("PartitionPumpInvokeProcessorCloseStart %r %r", - self.host.guid, self.partition_context.partition_id) + self.host.guid, self.partition_context.partition_id) except Exception as err: # pylint: disable=broad-except await self.process_error_async(err) _logger.error("%r %r %r", self.host.guid, self.partition_context.partition_id, err) From c8db7936362f5db68cbe8248b40a353d97c97425 Mon Sep 17 00:00:00 2001 From: annatisch Date: Wed, 22 Aug 2018 14:10:10 -0700 Subject: [PATCH 33/52] Renamed internal async module --- HISTORY.rst | 12 +++++++++++ README.rst | 5 +++++ azure/eventhub/__init__.py | 4 ++-- .../{_async => async_ops}/__init__.py | 6 +++--- .../{_async => async_ops}/receiver_async.py | 4 ++-- .../{_async => async_ops}/sender_async.py | 4 ++-- azure/eventhub/client.py | 20 +++++++++---------- azure/eventhub/common.py | 13 ++++++++++-- azure/eventhub/sender.py | 2 +- azure/eventprocessorhost/eh_partition_pump.py | 4 ++-- setup.py | 2 +- 11 files changed, 51 insertions(+), 25 deletions(-) rename azure/eventhub/{_async => async_ops}/__init__.py (98%) rename azure/eventhub/{_async => async_ops}/receiver_async.py (98%) rename azure/eventhub/{_async => async_ops}/sender_async.py (98%) diff --git a/HISTORY.rst b/HISTORY.rst index d60b724..6e90e55 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,18 @@ Release History =============== +1.0.0 (2018-08-22) +++++++++++++++++++ + +- API stable. +- Renamed internal `_async` module to `async_ops` for docs generation. +- Added optional `auth_timeout` parameter to `EventHubClient` and `EventHubClientAsync` to configure how long to allow for token + negotiation to complete. Default is 60 seconds. +- Added optional `send_timeout` parameter to `EventHubClient.add_sender` and `EventHubClientAsync.add_async_sender` to determine the + timeout for Events to be successfully sent. Default value is 60 seconds. +- Reformatted logging for performance. + + 0.2.0 (2018-08-06) ++++++++++++++++++ diff --git a/README.rst b/README.rst index 616771e..2d75dc3 100644 --- a/README.rst +++ b/README.rst @@ -26,6 +26,11 @@ Python 2.7 support The uAMQP library currently only supports Python 3.4 and above. Python 2.7 support is planned for a future release. +Documentation ++++++++++++++ +Reference documentation is available at `docs.microsoft.com/python/api/azure-eventhub `__. + + Examples +++++++++ diff --git a/azure/eventhub/__init__.py b/azure/eventhub/__init__.py index ae780c2..3cde06c 100644 --- a/azure/eventhub/__init__.py +++ b/azure/eventhub/__init__.py @@ -3,7 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -__version__ = "0.2.0" +__version__ = "1.0.0" from azure.eventhub.common import EventData, EventHubError, Offset from azure.eventhub.client import EventHubClient @@ -11,7 +11,7 @@ from azure.eventhub.receiver import Receiver try: - from azure.eventhub._async import ( + from azure.eventhub.async_ops import ( EventHubClientAsync, AsyncSender, AsyncReceiver) diff --git a/azure/eventhub/_async/__init__.py b/azure/eventhub/async_ops/__init__.py similarity index 98% rename from azure/eventhub/_async/__init__.py rename to azure/eventhub/async_ops/__init__.py index 5b102b5..7774724 100644 --- a/azure/eventhub/_async/__init__.py +++ b/azure/eventhub/async_ops/__init__.py @@ -183,7 +183,7 @@ def add_async_receiver( :operation: An optional operation to be appended to the hostname in the source URL. The value must start with `/` character. :type operation: str - :rtype: ~azure.eventhub._async.receiver_async.ReceiverAsync + :rtype: ~azure.eventhub.async_ops.receiver_async.ReceiverAsync """ path = self.address.path + operation if operation else self.address.path source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( @@ -214,7 +214,7 @@ def add_async_epoch_receiver( :operation: An optional operation to be appended to the hostname in the source URL. The value must start with `/` character. :type operation: str - :rtype: ~azure.eventhub._async.receiver_async.ReceiverAsync + :rtype: ~azure.eventhub.async_ops.receiver_async.ReceiverAsync """ path = self.address.path + operation if operation else self.address.path source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( @@ -249,7 +249,7 @@ def add_async_sender( :param auto_reconnect: Whether to automatically reconnect the sender if a retryable error occurs. Default value is `True`. :type auto_reconnect: bool - :rtype: ~azure.eventhub._async.sender_async.SenderAsync + :rtype: ~azure.eventhub.async_ops.sender_async.SenderAsync """ target = "amqps://{}{}".format(self.address.hostname, self.address.path) if operation: diff --git a/azure/eventhub/_async/receiver_async.py b/azure/eventhub/async_ops/receiver_async.py similarity index 98% rename from azure/eventhub/_async/receiver_async.py rename to azure/eventhub/async_ops/receiver_async.py index 0d49231..ad04520 100644 --- a/azure/eventhub/_async/receiver_async.py +++ b/azure/eventhub/async_ops/receiver_async.py @@ -29,7 +29,7 @@ def __init__( # pylint: disable=super-init-not-called Instantiate an async receiver. :param client: The parent EventHubClientAsync. - :type client: ~azure.eventhub._async.EventHubClientAsync + :type client: ~azure.eventhub.async_ops.EventHubClientAsync :param source: The source EventHub from which to receive events. :type source: ~uamqp.address.Source :param prefetch: The number of events to prefetch from the service @@ -78,7 +78,7 @@ async def open_async(self): context will be used to create a new handler before opening it. :param connection: The underlying client shared connection. - :type: connection: ~uamqp._async.connection_async.ConnectionAsync + :type: connection: ~uamqp.async_ops.connection_async.ConnectionAsync """ # pylint: disable=protected-access if self.redirected: diff --git a/azure/eventhub/_async/sender_async.py b/azure/eventhub/async_ops/sender_async.py similarity index 98% rename from azure/eventhub/_async/sender_async.py rename to azure/eventhub/async_ops/sender_async.py index bd72dd5..098c026 100644 --- a/azure/eventhub/_async/sender_async.py +++ b/azure/eventhub/async_ops/sender_async.py @@ -29,7 +29,7 @@ def __init__( # pylint: disable=super-init-not-called Instantiate an EventHub event SenderAsync handler. :param client: The parent EventHubClientAsync. - :type client: ~azure.eventhub._async.EventHubClientAsync + :type client: ~azure.eventhub.async_ops.EventHubClientAsync :param target: The URI of the EventHub to send to. :type target: str :param partition: The specific partition ID to send to. Default is `None`, in which case the service @@ -80,7 +80,7 @@ async def open_async(self): context will be used to create a new handler before opening it. :param connection: The underlying client shared connection. - :type: connection:~uamqp._async.connection_async.ConnectionAsync + :type: connection: ~uamqp.async_ops.connection_async.ConnectionAsync """ if self.redirected: self.target = self.redirected.address diff --git a/azure/eventhub/client.py b/azure/eventhub/client.py index 3a61c60..43c3b65 100644 --- a/azure/eventhub/client.py +++ b/azure/eventhub/client.py @@ -109,7 +109,7 @@ def __init__(self, address, username=None, password=None, debug=False, http_prox Additionally the following keys may also be present: 'username', 'password'. :type http_proxy: dict[str, Any] :param auth_timeout: The time in seconds to wait for a token to be authorized by the service. - The default value is 60 seconds. + The default value is 60 seconds. If set to 0, no timeout will be enforced from the client. :type auth_timeout: int """ self.container_id = "eventhub.pysdk-" + str(uuid.uuid4())[:8] @@ -142,6 +142,7 @@ def from_connection_string(cls, conn_str, eventhub=None, **kwargs): :type conn_str: str :param eventhub: The name of the EventHub, if the EntityName is not included in the connection string. + :type eventhub: str :param debug: Whether to output network trace logs to the logger. Default is `False`. :type debug: bool @@ -150,7 +151,7 @@ def from_connection_string(cls, conn_str, eventhub=None, **kwargs): Additionally the following keys may also be present: 'username', 'password'. :type http_proxy: dict[str, Any] :param auth_timeout: The time in seconds to wait for a token to be authorized by the service. - The default value is 60 seconds. + The default value is 60 seconds. If set to 0, no timeout will be enforced from the client. :type auth_timeout: int """ address, policy, key, entity = _parse_conn_str(conn_str) @@ -173,7 +174,7 @@ def from_iothub_connection_string(cls, conn_str, **kwargs): Additionally the following keys may also be present: 'username', 'password'. :type http_proxy: dict[str, Any] :param auth_timeout: The time in seconds to wait for a token to be authorized by the service. - The default value is 60 seconds. + The default value is 60 seconds. If set to 0, no timeout will be enforced from the client. :type auth_timeout: int """ address, policy, key, _ = _parse_conn_str(conn_str) @@ -297,11 +298,11 @@ def get_eventhub_info(self): """ Get details on the specified EventHub. Keys in the details dictionary include: - -'name' - -'type' - -'created_at' - -'partition_count' - -'partition_ids' + -'name' + -'type' + -'created_at' + -'partition_count' + -'partition_ids' :rtype: dict """ @@ -394,8 +395,7 @@ def add_epoch_receiver( def add_sender(self, partition=None, operation=None, send_timeout=60, keep_alive=30, auto_reconnect=True): """ - Add a sender to the client to send ~azure.eventhub.common.EventData object - to an EventHub. + Add a sender to the client to EventData object to an EventHub. :param partition: Optionally specify a particular partition to send to. If omitted, the events will be distributed to available partitions via diff --git a/azure/eventhub/common.py b/azure/eventhub/common.py index 035a812..af4db4e 100644 --- a/azure/eventhub/common.py +++ b/azure/eventhub/common.py @@ -45,7 +45,7 @@ def _error_handler(error): class EventData(object): """ The EventData class is a holder of event content. - Acts as a wrapper to an ~uamqp.message.Message object. + Acts as a wrapper to an uamqp.message.Message object. """ PROP_SEQ_NUMBER = b"x-opt-sequence-number" @@ -186,7 +186,7 @@ def body(self): """ The body of the event data object. - :rtype: bytes or generator[bytes] + :rtype: bytes or Generator[bytes] """ return self.message.get_data() @@ -194,6 +194,7 @@ def body(self): class Offset(object): """ The offset (position or timestamp) where a receiver starts. Examples: + Beginning of the event stream: >>> offset = Offset("-1") End of the event stream: @@ -238,6 +239,14 @@ def selector(self): class EventHubError(Exception): """ Represents an error happened in the client. + + :ivar message: The error message. + :vartype message: str + :ivar error: The error condition, if available. + :vartype error: str + :ivar details: The error details, if included in the + service response. + :vartype details: dict[str, str] """ def __init__(self, message, details=None): diff --git a/azure/eventhub/sender.py b/azure/eventhub/sender.py index e0ed738..b7fef5e 100644 --- a/azure/eventhub/sender.py +++ b/azure/eventhub/sender.py @@ -235,7 +235,7 @@ def transfer(self, event_data, callback=None): :type event_data: ~azure.eventhub.common.EventData :param callback: Callback to be run once the message has been send. This must be a function that accepts two arguments. - :type callback: func[~uamqp.constants.MessageSendResult, ~azure.eventhub.common.EventHubError] + :type callback: callable[~uamqp.constants.MessageSendResult, ~azure.eventhub.common.EventHubError] """ if self.error: raise self.error diff --git a/azure/eventprocessorhost/eh_partition_pump.py b/azure/eventprocessorhost/eh_partition_pump.py index 995440a..4ebd6a9 100644 --- a/azure/eventprocessorhost/eh_partition_pump.py +++ b/azure/eventprocessorhost/eh_partition_pump.py @@ -36,7 +36,7 @@ async def on_open_async(self): _opened_ok = True except Exception as err: # pylint: disable=broad-except _logger.warning( - "%r,%r PartitionPumpWarning: Failure creating client or receiver, " + "%r,%r PartitionPumpWarning: Failure creating client or receiver, " + "retrying: %r", self.host.guid, self.partition_context.partition_id, err) last_exception = err _retry_count += 1 @@ -91,7 +91,7 @@ async def clean_up_clients_async(self): async def on_closing_async(self, reason): """ - Overides partition pump on cleasing. + Overides partition pump on closing. :param reason: The reason for the shutdown. :type reason: str diff --git a/setup.py b/setup.py index 8efb8aa..891a80e 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ zip_safe=False, packages=find_packages(exclude=["examples", "tests"]), install_requires=[ - 'uamqp>=0.2.1,<0.3.0', + 'uamqp>=1.0.0,<2.0.0', 'msrestazure~=0.4.11', 'azure-common~=1.1', 'azure-storage~=0.36.0' From f428531e8fa1268769b91efbee9e21e3f753c397 Mon Sep 17 00:00:00 2001 From: annatisch Date: Mon, 27 Aug 2018 07:49:38 -0700 Subject: [PATCH 34/52] Updated send example to match recv Fix for issue #56 --- examples/send.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/send.py b/examples/send.py index fe1d91c..6881b0d 100644 --- a/examples/send.py +++ b/examples/send.py @@ -32,7 +32,7 @@ raise ValueError("No EventHubs URL supplied.") client = EventHubClient(ADDRESS, debug=False, username=USER, password=KEY) - sender = client.add_sender(partition="1") + sender = client.add_sender(partition="0") client.run() try: start_time = time.time() From c5be0bb4214e30f791d5b9e38bbcdce7c96c594a Mon Sep 17 00:00:00 2001 From: annatisch Date: Tue, 11 Sep 2018 13:51:51 -0700 Subject: [PATCH 35/52] Added build badge to readme --- README.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.rst b/README.rst index 2d75dc3..aefe16e 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,16 @@ Microsoft Azure SDK for Event Hubs ================================== +.. image:: https://img.shields.io/pypi/v/azure-eventhub.svg?maxAge=2592000 + :target: https://pypi.python.org/pypi/azure-eventhub/ + +.. image:: https://img.shields.io/pypi/pyversions/azure-eventhub.svg?maxAge=2592000 + :target: https://pypi.python.org/pypi/azure-eventhub/ + +.. image:: https://travis-ci.org/Azure/azure-event-hubs-python.svg?branch=master + :target: https://travis-ci.org/Azure/azure-event-hubs-python + + A Python AMQP client for Azure Event Hubs the provides: - A sender to publish events to the Event Hubs service. From 72fc34857cfb0a62bb8dcb9fdf5cd08968646d57 Mon Sep 17 00:00:00 2001 From: annatisch Date: Thu, 20 Sep 2018 09:51:48 -0700 Subject: [PATCH 36/52] Fix for repeat startup --- azure/eventhub/async_ops/__init__.py | 3 ++- azure/eventhub/async_ops/receiver_async.py | 3 +++ azure/eventhub/async_ops/sender_async.py | 3 +++ azure/eventhub/client.py | 3 ++- azure/eventhub/receiver.py | 3 +++ azure/eventhub/sender.py | 3 +++ 6 files changed, 16 insertions(+), 2 deletions(-) diff --git a/azure/eventhub/async_ops/__init__.py b/azure/eventhub/async_ops/__init__.py index 7774724..7c62b3d 100644 --- a/azure/eventhub/async_ops/__init__.py +++ b/azure/eventhub/async_ops/__init__.py @@ -75,7 +75,8 @@ async def _wait_for_client(self, client): async def _start_client_async(self, client): try: - await client.open_async() + if not client.running: + await client.open_async() except Exception as exp: # pylint: disable=broad-except log.info("Encountered error while starting handler: %r", exp) await client.close_async(exception=exp) diff --git a/azure/eventhub/async_ops/receiver_async.py b/azure/eventhub/async_ops/receiver_async.py index ad04520..066915f 100644 --- a/azure/eventhub/async_ops/receiver_async.py +++ b/azure/eventhub/async_ops/receiver_async.py @@ -40,6 +40,7 @@ def __init__( # pylint: disable=super-init-not-called :param loop: An event loop. """ self.loop = loop or asyncio.get_event_loop() + self.running = False self.client = client self.source = source self.offset = offset @@ -81,6 +82,7 @@ async def open_async(self): :type: connection: ~uamqp.async_ops.connection_async.ConnectionAsync """ # pylint: disable=protected-access + self.running = True if self.redirected: self.source = self.redirected.address source = Source(self.source) @@ -188,6 +190,7 @@ async def close_async(self, exception=None): due to an error. :type exception: Exception """ + self.running = False if self.error: return elif isinstance(exception, errors.LinkRedirect): diff --git a/azure/eventhub/async_ops/sender_async.py b/azure/eventhub/async_ops/sender_async.py index 098c026..be53b1d 100644 --- a/azure/eventhub/async_ops/sender_async.py +++ b/azure/eventhub/async_ops/sender_async.py @@ -47,6 +47,7 @@ def __init__( # pylint: disable=super-init-not-called :param loop: An event loop. If not specified the default event loop will be used. """ self.loop = loop or asyncio.get_event_loop() + self.running = False self.client = client self.target = target self.partition = partition @@ -82,6 +83,7 @@ async def open_async(self): :param connection: The underlying client shared connection. :type: connection: ~uamqp.async_ops.connection_async.ConnectionAsync """ + self.running = True if self.redirected: self.target = self.redirected.address self._handler = SendClientAsync( @@ -173,6 +175,7 @@ async def close_async(self, exception=None): due to an error. :type exception: Exception """ + self.running = False if self.error: return elif isinstance(exception, errors.LinkRedirect): diff --git a/azure/eventhub/client.py b/azure/eventhub/client.py index 43c3b65..73abad5 100644 --- a/azure/eventhub/client.py +++ b/azure/eventhub/client.py @@ -233,7 +233,8 @@ def _close_clients(self): def _start_clients(self): for client in self.clients: try: - client.open() + if not client.running: + client.open() except Exception as exp: # pylint: disable=broad-except client.close(exception=exp) diff --git a/azure/eventhub/receiver.py b/azure/eventhub/receiver.py index 0b7b8a9..ebbb2b1 100644 --- a/azure/eventhub/receiver.py +++ b/azure/eventhub/receiver.py @@ -35,6 +35,7 @@ def __init__(self, client, source, offset=None, prefetch=300, epoch=None, keep_a :param epoch: An optional epoch value. :type epoch: int """ + self.running = False self.client = client self.source = source self.offset = offset @@ -75,6 +76,7 @@ def open(self): :type: connection: ~uamqp.connection.Connection """ # pylint: disable=protected-access + self.running = True if self.redirected: self.source = self.redirected.address source = Source(self.source) @@ -185,6 +187,7 @@ def close(self, exception=None): due to an error. :type exception: Exception """ + self.running = False if self.error: return elif isinstance(exception, errors.LinkRedirect): diff --git a/azure/eventhub/sender.py b/azure/eventhub/sender.py index b7fef5e..3f23885 100644 --- a/azure/eventhub/sender.py +++ b/azure/eventhub/sender.py @@ -40,6 +40,7 @@ def __init__(self, client, target, partition=None, send_timeout=60, keep_alive=N Default value is `True`. :type auto_reconnect: bool """ + self.running = False self.client = client self.target = target self.partition = partition @@ -74,6 +75,7 @@ def open(self): :param connection: The underlying client shared connection. :type: connection: ~uamqp.connection.Connection """ + self.running = True if self.redirected: self.target = self.redirected.address self._handler = SendClient( @@ -169,6 +171,7 @@ def close(self, exception=None): due to an error. :type exception: Exception """ + self.running = False if self.error: return elif isinstance(exception, errors.LinkRedirect): From 01b89986fe9b7fb08401b2fa3fb538822148b05e Mon Sep 17 00:00:00 2001 From: annatisch Date: Thu, 20 Sep 2018 11:56:11 -0700 Subject: [PATCH 37/52] Added more storage connect options to EPH --- .../azure_storage_checkpoint_manager.py | 36 ++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/azure/eventprocessorhost/azure_storage_checkpoint_manager.py b/azure/eventprocessorhost/azure_storage_checkpoint_manager.py index 8ac3abe..a749bf2 100644 --- a/azure/eventprocessorhost/azure_storage_checkpoint_manager.py +++ b/azure/eventprocessorhost/azure_storage_checkpoint_manager.py @@ -28,14 +28,39 @@ class AzureStorageCheckpointLeaseManager(AbstractCheckpointManager, AbstractLeas Manages checkpoints and lease with azure storage blobs. In this implementation, checkpoints are data that's actually in the lease blob, so checkpoint operations turn into lease operations under the covers. + + :param str storage_account_name: The storage account name. This is used to + authenticate requests signed with an account key and to construct the storage + endpoint. It is required unless a connection string is given. + :param str storage_account_key: The storage account key. This is used for shared key + authentication. If neither account key or sas token is specified, anonymous access + will be used. + :param str lease_container_name: The name of the container that will be used to store + leases. If it does not already exist it will be created. Default value is 'eph-leases'. + :param int lease_renew_interval: The interval in seconds at which EPH will attempt to + renew the lease of a particular partition. Default value is 10. + :param int lease_duration: The duration in seconds of a lease on a partition. + Default value is 30. + :param str sas_token: A shared access signature token to use to authenticate requests + instead of the account key. If account key and sas token are both specified, + account key will be used to sign. If neither are specified, anonymous access will be used. + :param str endpoint_suffix: The host base component of the url, minus the account name. + Defaults to Azure (core.windows.net). Override this to use a National Cloud. + :param str connection_string: If specified, this will override all other endpoint parameters. + See http://azure.microsoft.com/en-us/documentation/articles/storage-configure-connection-string/ + for the connection string format. """ - def __init__(self, storage_account_name, storage_account_key, lease_container_name, - storage_blob_prefix=None, lease_renew_interval=10, lease_duration=30): + def __init__(self, storage_account_name=None, storage_account_key=None, lease_container_name="eph-leases", + storage_blob_prefix=None, lease_renew_interval=10, lease_duration=30, + sas_token=None, endpoint_suffix="core.windows.net", connection_string=None): AbstractCheckpointManager.__init__(self) AbstractLeaseManager.__init__(self, lease_renew_interval, lease_duration) self.storage_account_name = storage_account_name self.storage_account_key = storage_account_key + self.storage_sas_token = sas_token + self.endpoint_suffix = endpoint_suffix + self.connection_string = connection_string self.lease_container_name = lease_container_name self.storage_blob_prefix = storage_blob_prefix self.storage_client = None @@ -47,8 +72,8 @@ def __init__(self, storage_account_name, storage_account_key, lease_container_na self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=32) # Validate storage inputs - if not self.storage_account_name or not self.storage_account_key: - raise ValueError("Need a valid storage account name and key") + if not self.storage_account_name and not self.connection_string: + raise ValueError("Need a valid storage account name or connection string.") if not re.compile(r"^[a-z0-9](([a-z0-9\-[^\-])){1,61}[a-z0-9]$").match(self.lease_container_name): raise ValueError("Azure Storage lease container name is invalid.\ Please check naming conventions at\ @@ -68,6 +93,9 @@ def initialize(self, host): self.host = host self.storage_client = BlockBlobService(account_name=self.storage_account_name, account_key=self.storage_account_key, + sas_token=self.storage_sas_token, + endpoint_suffix=self.endpoint_suffix, + connection_string=self.connection_string, request_session=self.request_session) self.consumer_group_directory = self.storage_blob_prefix + self.host.eh_config.consumer_group From d33c244e5da2a22fad0b20cdc11a737dce9c260c Mon Sep 17 00:00:00 2001 From: annatisch Date: Thu, 20 Sep 2018 11:56:38 -0700 Subject: [PATCH 38/52] Bumped version --- HISTORY.rst | 14 ++++++++++++++ azure/eventhub/__init__.py | 2 +- setup.py | 3 ++- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 6e90e55..1d5a4d0 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,20 @@ Release History =============== +1.1.0 (2018-09-21) +++++++++++++++++++ + +- Changes to `AzureStorageCheckpointLeaseManager` parameters to support other connection options (issue #61): + + - The `storage_account_name`, `storage_account_key` and `lease_container_name` arguments are now optional keyword arguments. + - Added a `sas_token` argument that must be specified with `storage_account_name` in place of `storage_account_key`. + - Added an `endpoint_suffix` argument to support storage endpoints in National Clouds. + - Added a `connection_string` argument that, if specified, overrides all other endpoint arguments. + - The `lease_container_name` argument now defaults to `"eph-leases"` if not specified. + +- Fix for clients failing to start if run called multipled times (issue #64). + + 1.0.0 (2018-08-22) ++++++++++++++++++ diff --git a/azure/eventhub/__init__.py b/azure/eventhub/__init__.py index 3cde06c..e07a603 100644 --- a/azure/eventhub/__init__.py +++ b/azure/eventhub/__init__.py @@ -3,7 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -__version__ = "1.0.0" +__version__ = "1.1.0" from azure.eventhub.common import EventData, EventHubError, Offset from azure.eventhub.client import EventHubClient diff --git a/setup.py b/setup.py index 891a80e..d95ac21 100644 --- a/setup.py +++ b/setup.py @@ -44,12 +44,13 @@ author_email='azpysdkhelp@microsoft.com', url='https://github.com/Azure/azure-event-hubs-python', classifiers=[ - 'Development Status :: 3 - Alpha', + 'Development Status :: 5 - Production/Stable', 'Programming Language :: Python', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'License :: OSI Approved :: MIT License', ], zip_safe=False, From 514f8264b3c41ee2ac7cb3dda5df5a39eaa6baff Mon Sep 17 00:00:00 2001 From: annatisch Date: Fri, 21 Sep 2018 11:32:57 -0700 Subject: [PATCH 39/52] Handler blocked until client started --- azure/eventhub/async_ops/receiver_async.py | 2 ++ azure/eventhub/async_ops/sender_async.py | 4 ++++ azure/eventhub/receiver.py | 2 ++ azure/eventhub/sender.py | 6 ++++++ tests/__init__.py | 2 +- tests/test_eph.py | 17 ----------------- tests/test_reconnect.py | 11 ++++++++++- 7 files changed, 25 insertions(+), 19 deletions(-) delete mode 100644 tests/test_eph.py diff --git a/azure/eventhub/async_ops/receiver_async.py b/azure/eventhub/async_ops/receiver_async.py index 066915f..c69681e 100644 --- a/azure/eventhub/async_ops/receiver_async.py +++ b/azure/eventhub/async_ops/receiver_async.py @@ -219,6 +219,8 @@ async def receive(self, max_batch_size=None, timeout=None): """ if self.error: raise self.error + if not self.running: + raise ValueError("Unable to receive until client has been started.") data_batch = [] try: timeout_ms = 1000 * timeout if timeout else 0 diff --git a/azure/eventhub/async_ops/sender_async.py b/azure/eventhub/async_ops/sender_async.py index be53b1d..4579602 100644 --- a/azure/eventhub/async_ops/sender_async.py +++ b/azure/eventhub/async_ops/sender_async.py @@ -202,6 +202,8 @@ async def send(self, event_data): """ if self.error: raise self.error + if not self.running: + raise ValueError("Unable to send until client has been started.") if event_data.partition_key and self.partition: raise ValueError("EventData partition key cannot be used with a partition sender.") event_data.message.on_send_complete = self._on_outcome @@ -241,6 +243,8 @@ async def wait_async(self): """ if self.error: raise self.error + if not self.running: + raise ValueError("Unable to send until client has been started.") try: await self._handler.wait_async() except (errors.LinkDetach, errors.ConnectionClose) as shutdown: diff --git a/azure/eventhub/receiver.py b/azure/eventhub/receiver.py index ebbb2b1..f7724a7 100644 --- a/azure/eventhub/receiver.py +++ b/azure/eventhub/receiver.py @@ -226,6 +226,8 @@ def receive(self, max_batch_size=None, timeout=None): """ if self.error: raise self.error + if not self.running: + raise ValueError("Unable to receive until client has been started.") data_batch = [] try: timeout_ms = 1000 * timeout if timeout else 0 diff --git a/azure/eventhub/sender.py b/azure/eventhub/sender.py index 3f23885..a73dfdd 100644 --- a/azure/eventhub/sender.py +++ b/azure/eventhub/sender.py @@ -198,6 +198,8 @@ def send(self, event_data): """ if self.error: raise self.error + if not self.running: + raise ValueError("Unable to send until client has been started.") if event_data.partition_key and self.partition: raise ValueError("EventData partition key cannot be used with a partition sender.") event_data.message.on_send_complete = self._on_outcome @@ -242,6 +244,8 @@ def transfer(self, event_data, callback=None): """ if self.error: raise self.error + if not self.running: + raise ValueError("Unable to send until client has been started.") if event_data.partition_key and self.partition: raise ValueError("EventData partition key cannot be used with a partition sender.") if callback: @@ -254,6 +258,8 @@ def wait(self): """ if self.error: raise self.error + if not self.running: + raise ValueError("Unable to send until client has been started.") try: self._handler.wait() except (errors.LinkDetach, errors.ConnectionClose) as shutdown: diff --git a/tests/__init__.py b/tests/__init__.py index 7ec7d3b..7b7c91a 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -12,7 +12,7 @@ def get_logger(filename, level=logging.INFO): azure_logger = logging.getLogger("azure") azure_logger.setLevel(level) uamqp_logger = logging.getLogger("uamqp") - uamqp_logger.setLevel(logging.DEBUG) + uamqp_logger.setLevel(logging.INFO) formatter = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s') console_handler = logging.StreamHandler(stream=sys.stdout) diff --git a/tests/test_eph.py b/tests/test_eph.py deleted file mode 100644 index c1d43e7..0000000 --- a/tests/test_eph.py +++ /dev/null @@ -1,17 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# ----------------------------------------------------------------------------------- - -import asyncio -import pytest - - -def test_eph_start(eph): - """ - Test that the processing host starts correctly - """ - pytest.skip("Not working yet") - loop = asyncio.get_event_loop() - loop.run_until_complete(eph.open_async()) - loop.run_until_complete(eph.close_async()) diff --git a/tests/test_reconnect.py b/tests/test_reconnect.py index a6aa0ce..bd10bbd 100644 --- a/tests/test_reconnect.py +++ b/tests/test_reconnect.py @@ -85,6 +85,15 @@ def test_send_with_forced_conn_close_sync(connection_str, receivers): assert list(received[0].body)[0] == b"A single event" +def pump(receiver): + messages = [] + batch = receiver.receive(timeout=1) + messages.extend(batch) + while batch: + batch = receiver.receive(timeout=1) + messages.extend(batch) + return messages + @pytest.mark.asyncio async def test_send_with_forced_conn_close_async(connection_str, receivers): #pytest.skip("long running") @@ -106,7 +115,7 @@ async def test_send_with_forced_conn_close_async(connection_str, receivers): received = [] for r in receivers: - received.extend(r.receive(timeout=1)) + received.extend(pump(r)) assert len(received) == 5 assert list(received[0].body)[0] == b"A single event" From 2fa535a121fb8e7df034e33d0552cd9849eb485e Mon Sep 17 00:00:00 2001 From: annatisch Date: Fri, 21 Sep 2018 12:58:04 -0700 Subject: [PATCH 40/52] Added event data methods --- .gitignore | 3 +++ HISTORY.rst | 1 + azure/eventhub/common.py | 42 +++++++++++++++++++++++++++-- examples/recv.py | 14 ++++++---- tests/test_negative.py | 58 +++++++++++++++++++++++++++++++++++++++- tests/test_receive.py | 10 +++++++ 6 files changed, 120 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 6b8bf2d..0785980 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,9 @@ pip-delete-this-directory.txt azure/storage/ azure/common/ azure/profiles/ +*.log.1 +*.log.2 +*.log.3 htmlcov/ .tox/ diff --git a/HISTORY.rst b/HISTORY.rst index 1d5a4d0..c77b3c7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -15,6 +15,7 @@ Release History - The `lease_container_name` argument now defaults to `"eph-leases"` if not specified. - Fix for clients failing to start if run called multipled times (issue #64). +- Added convenience methods `body_as_str` and `body_as_json` to EventData object for easier processing of message data. 1.0.0 (2018-08-22) diff --git a/azure/eventhub/common.py b/azure/eventhub/common.py index af4db4e..1c7d176 100644 --- a/azure/eventhub/common.py +++ b/azure/eventhub/common.py @@ -5,6 +5,7 @@ import datetime import time +import json from uamqp import Message, BatchMessage from uamqp import types, constants, errors @@ -88,7 +89,6 @@ def __init__(self, body=None, batch=None, to_device=None, message=None): else: self.message = Message(body, properties=self.msg_properties) - @property def sequence_number(self): """ @@ -188,7 +188,45 @@ def body(self): :rtype: bytes or Generator[bytes] """ - return self.message.get_data() + try: + return self.message.get_data() + except TypeError: + raise ValueError("Message data empty.") + + def body_as_str(self, encoding='UTF-8'): + """ + The body of the event data as a string if the data is of a + compatible type. + + :param encoding: The encoding to use for decoding message data. + Default is 'UTF-8' + :rtype: str + """ + data = self.body + try: + return "".join(b.decode(encoding) for b in data) + except TypeError: + return str(data) + except: + pass + try: + return data.decode(encoding) + except Exception as e: + raise TypeError("Message data is not compatible with string type: {}".format(e)) + + def body_as_json(self, encoding='UTF-8'): + """ + The body of the event loaded as a JSON object is the data is compatible. + + :param encoding: The encoding to use for decoding message data. + Default is 'UTF-8' + :rtype: dict + """ + data_str = self.body_as_str() + try: + return json.loads(data_str) + except Exception as e: + raise TypeError("Event data is not compatible with JSON type: {}".format(e)) class Offset(object): diff --git a/examples/recv.py b/examples/recv.py index d2fbdf7..f43d03b 100644 --- a/examples/recv.py +++ b/examples/recv.py @@ -38,11 +38,15 @@ receiver = client.add_receiver(CONSUMER_GROUP, PARTITION, prefetch=5000, offset=OFFSET) client.run() start_time = time.time() - for event_data in receiver.receive(timeout=100): - last_offset = event_data.offset - last_sn = event_data.sequence_number - print("Received: {}, {}".format(last_offset.value, last_sn)) - total += 1 + batch = receiver.receive(timeout=5000) + while batch: + for event_data in batch: + last_offset = event_data.offset + last_sn = event_data.sequence_number + print("Received: {}, {}".format(last_offset.value, last_sn)) + print(event_data.body_as_str()) + total += 1 + batch = receiver.receive(timeout=5000) end_time = time.time() client.stop() diff --git a/tests/test_negative.py b/tests/test_negative.py index dbc8096..bdfcbfd 100644 --- a/tests/test_negative.py +++ b/tests/test_negative.py @@ -7,6 +7,7 @@ import os import asyncio import pytest +import time from azure import eventhub from azure.eventhub import ( @@ -303,4 +304,59 @@ async def test_max_receivers_async(connection_str, senders): assert len(failed) == 1 print(failed[0].message) finally: - await client.stop_async() \ No newline at end of file + await client.stop_async() + + +def test_message_body_types(connection_str, senders): + client = EventHubClient.from_connection_string(connection_str, debug=False) + receiver = client.add_receiver("$default", "0", offset=Offset('@latest')) + try: + client.run() + + received = receiver.receive(timeout=5) + assert len(received) == 0 + senders[0].send(EventData(b"Bytes Data")) + time.sleep(1) + received = receiver.receive(timeout=5) + assert len(received) == 1 + assert list(received[0].body) == [b'Bytes Data'] + assert received[0].body_as_str() == "Bytes Data" + with pytest.raises(TypeError): + received[0].body_as_json() + + senders[0].send(EventData("Str Data")) + time.sleep(1) + received = receiver.receive(timeout=5) + assert len(received) == 1 + assert list(received[0].body) == [b'Str Data'] + assert received[0].body_as_str() == "Str Data" + with pytest.raises(TypeError): + received[0].body_as_json() + + senders[0].send(EventData(b'{"test_value": "JSON bytes data", "key1": true, "key2": 42}')) + time.sleep(1) + received = receiver.receive(timeout=5) + assert len(received) == 1 + assert list(received[0].body) == [b'{"test_value": "JSON bytes data", "key1": true, "key2": 42}'] + assert received[0].body_as_str() == '{"test_value": "JSON bytes data", "key1": true, "key2": 42}' + assert received[0].body_as_json() == {"test_value": "JSON bytes data", "key1": True, "key2": 42} + + senders[0].send(EventData('{"test_value": "JSON str data", "key1": true, "key2": 42}')) + time.sleep(1) + received = receiver.receive(timeout=5) + assert len(received) == 1 + assert list(received[0].body) == [b'{"test_value": "JSON str data", "key1": true, "key2": 42}'] + assert received[0].body_as_str() == '{"test_value": "JSON str data", "key1": true, "key2": 42}' + assert received[0].body_as_json() == {"test_value": "JSON str data", "key1": True, "key2": 42} + + senders[0].send(EventData(42)) + time.sleep(1) + received = receiver.receive(timeout=5) + assert len(received) == 1 + assert received[0].body_as_str() == "42" + with pytest.raises(ValueError): + received[0].body + except: + raise + finally: + client.stop() \ No newline at end of file diff --git a/tests/test_receive.py b/tests/test_receive.py index 1b7480e..1bdbe86 100644 --- a/tests/test_receive.py +++ b/tests/test_receive.py @@ -24,6 +24,7 @@ def test_receive_end_of_stream(connection_str, senders): received = receiver.receive(timeout=5) assert len(received) == 1 + assert received[0].body_as_str() == "Receiving only a single event" assert list(received[-1].body)[0] == b"Receiving only a single event" except: raise @@ -48,6 +49,9 @@ def test_receive_with_offset_sync(connection_str, senders): assert len(received) == 1 offset = received[0].offset + assert list(received[0].body) == [b'Data'] + assert received[0].body_as_str() == "Data" + offset_receiver = client.add_receiver("$default", "0", offset=offset) client.run() received = offset_receiver.receive(timeout=5) @@ -75,6 +79,9 @@ def test_receive_with_inclusive_offset(connection_str, senders): assert len(received) == 1 offset = received[0].offset + assert list(received[0].body) == [b'Data'] + assert received[0].body_as_str() == "Data" + offset_receiver = client.add_receiver("$default", "0", offset=Offset(offset.value, inclusive=True)) client.run() received = offset_receiver.receive(timeout=5) @@ -101,6 +108,9 @@ def test_receive_with_datetime(connection_str, senders): assert len(received) == 1 offset = received[0].enqueued_time + assert list(received[0].body) == [b'Data'] + assert received[0].body_as_str() == "Data" + offset_receiver = client.add_receiver("$default", "0", offset=Offset(offset)) client.run() received = offset_receiver.receive(timeout=5) From 551d47db4a874d69165fbfc011a2dbfee912cfaa Mon Sep 17 00:00:00 2001 From: annatisch Date: Fri, 21 Sep 2018 13:06:12 -0700 Subject: [PATCH 41/52] Fix pylint --- .travis.yml | 10 +++++++++- azure/eventhub/common.py | 4 ++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 17f7c2e..5b2b8ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,15 @@ language: python cache: pip python: - "3.6" -# command to install dependencies +dist: xenial +matrix: + include: + - os: linux + python: "3.5" + - os: linux + python: "3.6" + - os: linux + python: "3.7" install: - pip install -r dev_requirements.txt - pip install -e . diff --git a/azure/eventhub/common.py b/azure/eventhub/common.py index 1c7d176..e63b639 100644 --- a/azure/eventhub/common.py +++ b/azure/eventhub/common.py @@ -207,7 +207,7 @@ def body_as_str(self, encoding='UTF-8'): return "".join(b.decode(encoding) for b in data) except TypeError: return str(data) - except: + except: # pylint: disable=bare-except pass try: return data.decode(encoding) @@ -222,7 +222,7 @@ def body_as_json(self, encoding='UTF-8'): Default is 'UTF-8' :rtype: dict """ - data_str = self.body_as_str() + data_str = self.body_as_str(encoding=encoding) try: return json.loads(data_str) except Exception as e: From 0a4cc2eae0f01c33c7203036de7fa2ccd389b105 Mon Sep 17 00:00:00 2001 From: annatisch Date: Fri, 21 Sep 2018 13:09:40 -0700 Subject: [PATCH 42/52] Fix 3.7 CI --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5b2b8ea..ac5bef4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,5 @@ language: python cache: pip -python: - - "3.6" dist: xenial matrix: include: From 9afbdbfbf561d458039ec45951b73c77d128a737 Mon Sep 17 00:00:00 2001 From: annatisch Date: Fri, 21 Sep 2018 13:28:52 -0700 Subject: [PATCH 43/52] Fix 3.7 CI --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index ac5bef4..6addbf5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,7 @@ language: python cache: pip dist: xenial +sudo: required matrix: include: - os: linux From b3910455851e953f1f58a23faa7ed880eebc4ec8 Mon Sep 17 00:00:00 2001 From: annatisch Date: Fri, 21 Sep 2018 13:46:34 -0700 Subject: [PATCH 44/52] Updated pylint version --- dev_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index 3cbeb9a..31a0ba7 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -2,5 +2,5 @@ pytest>=3.4.1 pytest-asyncio>=0.8.0 docutils>=0.14 pygments>=2.2.0 -pylint==1.8.4 +pylint==2.1.1 behave==1.2.6 \ No newline at end of file From da94d065bba10412bf6afd4cb7fbaac9662ff279 Mon Sep 17 00:00:00 2001 From: annatisch Date: Fri, 21 Sep 2018 14:51:16 -0700 Subject: [PATCH 45/52] Pylint fixes --- azure/eventhub/async_ops/receiver_async.py | 27 +++++++++---------- azure/eventhub/async_ops/sender_async.py | 9 +++---- azure/eventhub/client.py | 2 -- azure/eventhub/common.py | 10 +++---- azure/eventhub/receiver.py | 23 +++++++--------- azure/eventhub/sender.py | 9 +++---- azure/eventprocessorhost/eh_partition_pump.py | 4 +-- azure/eventprocessorhost/partition_context.py | 2 +- azure/eventprocessorhost/partition_pump.py | 2 +- pylintrc | 2 +- 10 files changed, 40 insertions(+), 50 deletions(-) diff --git a/azure/eventhub/async_ops/receiver_async.py b/azure/eventhub/async_ops/receiver_async.py index c69681e..814adff 100644 --- a/azure/eventhub/async_ops/receiver_async.py +++ b/azure/eventhub/async_ops/receiver_async.py @@ -173,12 +173,11 @@ async def has_started(self): timeout, auth_in_progress = await self._handler._auth.handle_token_async() if timeout: raise EventHubError("Authorization timeout.") - elif auth_in_progress: + if auth_in_progress: return False - elif not await self._handler._client_ready_async(): + if not await self._handler._client_ready_async(): return False - else: - return True + return True async def close_async(self, exception=None): """ @@ -193,7 +192,7 @@ async def close_async(self, exception=None): self.running = False if self.error: return - elif isinstance(exception, errors.LinkRedirect): + if isinstance(exception, errors.LinkRedirect): self.redirected = exception elif isinstance(exception, EventHubError): self.error = exception @@ -237,21 +236,19 @@ async def receive(self, max_batch_size=None, timeout=None): log.info("AsyncReceiver detached. Attempting reconnect.") await self.reconnect_async() return data_batch - else: - log.info("AsyncReceiver detached. Shutting down.") - error = EventHubError(str(shutdown), shutdown) - await self.close_async(exception=error) - raise error + log.info("AsyncReceiver detached. Shutting down.") + error = EventHubError(str(shutdown), shutdown) + await self.close_async(exception=error) + raise error except errors.MessageHandlerError as shutdown: if self.auto_reconnect: log.info("AsyncReceiver detached. Attempting reconnect.") await self.reconnect_async() return data_batch - else: - log.info("AsyncReceiver detached. Shutting down.") - error = EventHubError(str(shutdown), shutdown) - await self.close_async(exception=error) - raise error + log.info("AsyncReceiver detached. Shutting down.") + error = EventHubError(str(shutdown), shutdown) + await self.close_async(exception=error) + raise error except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) error = EventHubError("Receive failed: {}".format(e)) diff --git a/azure/eventhub/async_ops/sender_async.py b/azure/eventhub/async_ops/sender_async.py index 4579602..9f46fdd 100644 --- a/azure/eventhub/async_ops/sender_async.py +++ b/azure/eventhub/async_ops/sender_async.py @@ -158,12 +158,11 @@ async def has_started(self): timeout, auth_in_progress = await self._handler._auth.handle_token_async() if timeout: raise EventHubError("Authorization timeout.") - elif auth_in_progress: + if auth_in_progress: return False - elif not await self._handler._client_ready_async(): + if not await self._handler._client_ready_async(): return False - else: - return True + return True async def close_async(self, exception=None): """ @@ -178,7 +177,7 @@ async def close_async(self, exception=None): self.running = False if self.error: return - elif isinstance(exception, errors.LinkRedirect): + if isinstance(exception, errors.LinkRedirect): self.redirected = exception elif isinstance(exception, EventHubError): self.error = exception diff --git a/azure/eventhub/client.py b/azure/eventhub/client.py index 73abad5..06508df 100644 --- a/azure/eventhub/client.py +++ b/azure/eventhub/client.py @@ -330,8 +330,6 @@ def get_eventhub_info(self): output['partition_count'] = eh_info[b'partition_count'] output['partition_ids'] = [p.decode('utf-8') for p in eh_info[b'partition_ids']] return output - except: - raise finally: mgmt_client.close() diff --git a/azure/eventhub/common.py b/azure/eventhub/common.py index e63b639..b4a1755 100644 --- a/azure/eventhub/common.py +++ b/azure/eventhub/common.py @@ -32,13 +32,13 @@ def _error_handler(error): """ if error.condition == b'com.microsoft:server-busy': return errors.ErrorAction(retry=True, backoff=4) - elif error.condition == b'com.microsoft:timeout': + if error.condition == b'com.microsoft:timeout': return errors.ErrorAction(retry=True, backoff=2) - elif error.condition == b'com.microsoft:operation-cancelled': + if error.condition == b'com.microsoft:operation-cancelled': return errors.ErrorAction(retry=True) - elif error.condition == b"com.microsoft:container-close": + if error.condition == b"com.microsoft:container-close": return errors.ErrorAction(retry=True, backoff=4) - elif error.condition in _NO_RETRY_ERRORS: + if error.condition in _NO_RETRY_ERRORS: return errors.ErrorAction(retry=False) return errors.ErrorAction(retry=True) @@ -269,7 +269,7 @@ def selector(self): if isinstance(self.value, datetime.datetime): timestamp = (time.mktime(self.value.timetuple()) * 1000) + (self.value.microsecond/1000) return ("amqp.annotation.x-opt-enqueued-time {} '{}'".format(operator, int(timestamp))).encode('utf-8') - elif isinstance(self.value, int): + if isinstance(self.value, int): return ("amqp.annotation.x-opt-sequence-number {} '{}'".format(operator, self.value)).encode('utf-8') return ("amqp.annotation.x-opt-offset {} '{}'".format(operator, self.value)).encode('utf-8') diff --git a/azure/eventhub/receiver.py b/azure/eventhub/receiver.py index f7724a7..6822149 100644 --- a/azure/eventhub/receiver.py +++ b/azure/eventhub/receiver.py @@ -170,12 +170,11 @@ def has_started(self): timeout, auth_in_progress = self._handler._auth.handle_token() if timeout: raise EventHubError("Authorization timeout.") - elif auth_in_progress: + if auth_in_progress: return False - elif not self._handler._client_ready(): + if not self._handler._client_ready(): return False - else: - return True + return True def close(self, exception=None): """ @@ -190,7 +189,7 @@ def close(self, exception=None): self.running = False if self.error: return - elif isinstance(exception, errors.LinkRedirect): + if isinstance(exception, errors.LinkRedirect): self.redirected = exception elif isinstance(exception, EventHubError): self.error = exception @@ -243,18 +242,16 @@ def receive(self, max_batch_size=None, timeout=None): if shutdown.action.retry and self.auto_reconnect: self.reconnect() return data_batch - else: - error = EventHubError(str(shutdown), shutdown) - self.close(exception=error) - raise error + error = EventHubError(str(shutdown), shutdown) + self.close(exception=error) + raise error except errors.MessageHandlerError as shutdown: if self.auto_reconnect: self.reconnect() return data_batch - else: - error = EventHubError(str(shutdown), shutdown) - self.close(exception=error) - raise error + error = EventHubError(str(shutdown), shutdown) + self.close(exception=error) + raise error except Exception as e: error = EventHubError("Receive failed: {}".format(e)) self.close(exception=error) diff --git a/azure/eventhub/sender.py b/azure/eventhub/sender.py index a73dfdd..b4ed3b7 100644 --- a/azure/eventhub/sender.py +++ b/azure/eventhub/sender.py @@ -154,12 +154,11 @@ def has_started(self): timeout, auth_in_progress = self._handler._auth.handle_token() if timeout: raise EventHubError("Authorization timeout.") - elif auth_in_progress: + if auth_in_progress: return False - elif not self._handler._client_ready(): + if not self._handler._client_ready(): return False - else: - return True + return True def close(self, exception=None): """ @@ -174,7 +173,7 @@ def close(self, exception=None): self.running = False if self.error: return - elif isinstance(exception, errors.LinkRedirect): + if isinstance(exception, errors.LinkRedirect): self.redirected = exception elif isinstance(exception, EventHubError): self.error = exception diff --git a/azure/eventprocessorhost/eh_partition_pump.py b/azure/eventprocessorhost/eh_partition_pump.py index 4ebd6a9..e0aa25d 100644 --- a/azure/eventprocessorhost/eh_partition_pump.py +++ b/azure/eventprocessorhost/eh_partition_pump.py @@ -36,8 +36,8 @@ async def on_open_async(self): _opened_ok = True except Exception as err: # pylint: disable=broad-except _logger.warning( - "%r,%r PartitionPumpWarning: Failure creating client or receiver, " + - "retrying: %r", self.host.guid, self.partition_context.partition_id, err) + "%r,%r PartitionPumpWarning: Failure creating client or receiver, retrying: %r", + self.host.guid, self.partition_context.partition_id, err) last_exception = err _retry_count += 1 diff --git a/azure/eventprocessorhost/partition_context.py b/azure/eventprocessorhost/partition_context.py index 510fdd6..b33099e 100644 --- a/azure/eventprocessorhost/partition_context.py +++ b/azure/eventprocessorhost/partition_context.py @@ -121,7 +121,7 @@ async def persist_checkpoint_async(self, checkpoint): self.lease.offset = checkpoint.offset self.lease.sequence_number = checkpoint.sequence_number else: - _logger.error( + _logger.error( # pylint: disable=logging-not-lazy "Ignoring out of date checkpoint with offset %r/sequence number %r because " + "current persisted checkpoint has higher offset %r/sequence number %r", checkpoint.offset, diff --git a/azure/eventprocessorhost/partition_pump.py b/azure/eventprocessorhost/partition_pump.py index be8be04..cc2dcdc 100644 --- a/azure/eventprocessorhost/partition_pump.py +++ b/azure/eventprocessorhost/partition_pump.py @@ -143,7 +143,7 @@ async def process_events_async(self, events): # CloseAsync are protected by synchronizing too. try: last = events[-1] - if last != None: + if last is not None: self.partition_context.set_offset_and_sequence_number(last) await self.processor.process_events_async(self.partition_context, events) except Exception as err: # pylint: disable=broad-except diff --git a/pylintrc b/pylintrc index 7b3f956..6e495c8 100644 --- a/pylintrc +++ b/pylintrc @@ -6,7 +6,7 @@ reports=no # For all codes, run 'pylint --list-msgs' or go to 'https://pylint.readthedocs.io/en/latest/reference_guide/features.html' # locally-disabled: Warning locally suppressed using disable-msg # cyclic-import: because of https://github.com/PyCQA/pylint/issues/850 -disable=raising-bad-type,missing-docstring,locally-disabled,fixme,cyclic-import,too-many-arguments,invalid-name,duplicate-code,logging-format-interpolation,too-many-instance-attributes,too-few-public-methods +disable=useless-object-inheritance,raising-bad-type,missing-docstring,locally-disabled,fixme,cyclic-import,too-many-arguments,invalid-name,duplicate-code,logging-format-interpolation,too-many-instance-attributes,too-few-public-methods [FORMAT] max-line-length=120 From c9e3323a37a5aed2c9eb4f1d21b1694a4ed468ab Mon Sep 17 00:00:00 2001 From: annatisch Date: Fri, 28 Sep 2018 12:47:05 -0700 Subject: [PATCH 46/52] Made setup 2.7 compatible --- .travis.yml | 34 ++++++++++++++++++---- HISTORY.rst | 6 ++++ conftest.py | 69 ++++++++++---------------------------------- dev_requirements.txt | 5 ++-- setup.cfg | 2 ++ setup.py | 10 ++++--- 6 files changed, 62 insertions(+), 64 deletions(-) create mode 100644 setup.cfg diff --git a/.travis.yml b/.travis.yml index 6addbf5..37ac36e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,17 +4,41 @@ dist: xenial sudo: required matrix: include: + - os: linux + python: "2.7" + dist: trusty + script: + - pytest + - python ./setup.py check -r -s + - pylint --ignore=async_ops azure.eventhub + - os: linux + python: "3.4" + dist: trusty + script: + - pytest + - python ./setup.py check -r -s + - pylint --ignore=async_ops azure.eventhub - os: linux python: "3.5" + script: + - pytest + - python ./setup.py check -r -s + - pylint azure.eventhub + - pylint azure.eventprocessorhost - os: linux python: "3.6" + script: + - pytest + - python ./setup.py check -r -s + - pylint azure.eventhub + - pylint azure.eventprocessorhost - os: linux python: "3.7" + script: + - pytest + - python ./setup.py check -r -s + - pylint azure.eventhub + - pylint azure.eventprocessorhost install: - pip install -r dev_requirements.txt - pip install -e . -script: - - pytest - - python ./setup.py check -r -s - - pylint azure.eventhub - - pylint azure.eventprocessorhost diff --git a/HISTORY.rst b/HISTORY.rst index c77b3c7..56b246f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,12 @@ Release History =============== +1.2.0 (release-candidate) ++++++++++++++++++++++++++ + +- Support for Python 2.7. + + 1.1.0 (2018-09-21) ++++++++++++++++++ diff --git a/conftest.py b/conftest.py index 96d135d..2cf7966 100644 --- a/conftest.py +++ b/conftest.py @@ -9,18 +9,25 @@ import logging import sys +# Ignore async tests for Python < 3.5 +collect_ignore = [] +if sys.version_info < (3, 5): + collect_ignore.append("tests/asynctests") + collect_ignore.append("features") +else: + from tests.asynctests import MockEventProcessor + from azure.eventprocessorhost import EventProcessorHost + from azure.eventprocessorhost import EventHubPartitionPump + from azure.eventprocessorhost import AzureStorageCheckpointLeaseManager + from azure.eventprocessorhost import AzureBlobLease + from azure.eventprocessorhost import EventHubConfig + from azure.eventprocessorhost.lease import Lease + from azure.eventprocessorhost.partition_pump import PartitionPump + from azure.eventprocessorhost.partition_manager import PartitionManager + from tests import get_logger from azure import eventhub from azure.eventhub import EventHubClient, Receiver, Offset -from azure.eventprocessorhost import EventProcessorHost -from azure.eventprocessorhost import EventHubPartitionPump -from azure.eventprocessorhost import AzureStorageCheckpointLeaseManager -from azure.eventprocessorhost import AzureBlobLease -from azure.eventprocessorhost import EventHubConfig -from azure.eventprocessorhost.lease import Lease -from azure.eventprocessorhost.partition_pump import PartitionPump -from azure.eventprocessorhost.partition_manager import PartitionManager -from azure.eventprocessorhost.abstract_event_processor import AbstractEventProcessor log = get_logger(None, logging.DEBUG) @@ -181,47 +188,3 @@ def partition_pump(eph): def partition_manager(eph): partition_manager = PartitionManager(eph) return partition_manager - - -class MockEventProcessor(AbstractEventProcessor): - """ - Mock Implmentation of AbstractEventProcessor for testing - """ - def __init__(self, params=None): - """ - Init Event processor - """ - self.params = params - self._msg_counter = 0 - - async def open_async(self, context): - """ - Called by processor host to initialize the event processor. - """ - logging.info("Connection established {}".format(context.partition_id)) - - async def close_async(self, context, reason): - """ - Called by processor host to indicate that the event processor is being stopped. - (Params) Context:Information about the partition - """ - logging.info("Connection closed (reason {}, id {}, offset {}, sq_number {})".format( - reason, context.partition_id, context.offset, context.sequence_number)) - - async def process_events_async(self, context, messages): - """ - Called by the processor host when a batch of events has arrived. - This is where the real work of the event processor is done. - (Params) Context: Information about the partition, Messages: The events to be processed. - """ - logging.info("Events processed {} {}".format(context.partition_id, messages)) - await context.checkpoint_async() - - async def process_error_async(self, context, error): - """ - Called when the underlying client experiences an error while receiving. - EventProcessorHost will take care of recovering from the error and - continuing to pump messages,so no action is required from - (Params) Context: Information about the partition, Error: The error that occured. - """ - logging.error("Event Processor Error {!r}".format(error)) \ No newline at end of file diff --git a/dev_requirements.txt b/dev_requirements.txt index 31a0ba7..5d01ff4 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,6 +1,7 @@ pytest>=3.4.1 -pytest-asyncio>=0.8.0 +pytest-asyncio>=0.8.0; python_version > '3.4' docutils>=0.14 pygments>=2.2.0 -pylint==2.1.1 +pylint==2.1.1; python_version >= '3.4' +pylint==1.8.4; python_version < '3.4' behave==1.2.6 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..3480374 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal=1 \ No newline at end of file diff --git a/setup.py b/setup.py index d95ac21..cbb4d25 100644 --- a/setup.py +++ b/setup.py @@ -29,9 +29,9 @@ if not version: raise RuntimeError('Cannot find version information') -with open('README.rst', encoding='utf-8') as f: +with open('README.rst') as f: readme = f.read() -with open('HISTORY.rst', encoding='utf-8') as f: +with open('HISTORY.rst') as f: history = f.read() setup( @@ -46,6 +46,8 @@ classifiers=[ 'Development Status :: 5 - Production/Stable', 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', @@ -57,8 +59,8 @@ packages=find_packages(exclude=["examples", "tests"]), install_requires=[ 'uamqp>=1.0.0,<2.0.0', - 'msrestazure~=0.4.11', + 'msrestazure~=0.5', 'azure-common~=1.1', - 'azure-storage~=0.36.0' + 'azure-storage-blob~=1.3' ] ) From 3af1d7ab97e3d1e762cf95bdb690de290a36c560 Mon Sep 17 00:00:00 2001 From: annatisch Date: Mon, 1 Oct 2018 13:36:58 -0700 Subject: [PATCH 47/52] Separated async tests --- tests/asynctests/__init__.py | 54 +++++ .../test_checkpoint_manager.py | 0 .../test_eh_partition_pump.py | 0 .../test_iothub_receive_async.py | 0 tests/asynctests/test_longrunning_eph.py | 154 +++++++++++++++ .../test_longrunning_receive_async.py | 127 ++++++++++++ .../test_longrunning_send_async.py | 51 +++-- tests/asynctests/test_negative_async.py | 185 ++++++++++++++++++ .../test_partition_manager.py | 0 tests/{ => asynctests}/test_partition_pump.py | 0 tests/{ => asynctests}/test_receive_async.py | 0 tests/asynctests/test_reconnect_async.py | 90 +++++++++ tests/{ => asynctests}/test_send_async.py | 0 tests/test_longrunning_receive.py | 103 ++++++---- tests/test_longrunning_send.py | 27 ++- tests/test_negative.py | 170 ---------------- tests/test_reconnect.py | 58 ------ 17 files changed, 723 insertions(+), 296 deletions(-) create mode 100644 tests/asynctests/__init__.py rename tests/{ => asynctests}/test_checkpoint_manager.py (100%) rename tests/{ => asynctests}/test_eh_partition_pump.py (100%) rename tests/{ => asynctests}/test_iothub_receive_async.py (100%) create mode 100644 tests/asynctests/test_longrunning_eph.py create mode 100644 tests/asynctests/test_longrunning_receive_async.py rename tests/{ => asynctests}/test_longrunning_send_async.py (69%) create mode 100644 tests/asynctests/test_negative_async.py rename tests/{ => asynctests}/test_partition_manager.py (100%) rename tests/{ => asynctests}/test_partition_pump.py (100%) rename tests/{ => asynctests}/test_receive_async.py (100%) create mode 100644 tests/asynctests/test_reconnect_async.py rename tests/{ => asynctests}/test_send_async.py (100%) diff --git a/tests/asynctests/__init__.py b/tests/asynctests/__init__.py new file mode 100644 index 0000000..dc00a0f --- /dev/null +++ b/tests/asynctests/__init__.py @@ -0,0 +1,54 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +#-------------------------------------------------------------------------- + +import asyncio +import logging + +from azure.eventprocessorhost.abstract_event_processor import AbstractEventProcessor + + +class MockEventProcessor(AbstractEventProcessor): + """ + Mock Implmentation of AbstractEventProcessor for testing + """ + def __init__(self, params=None): + """ + Init Event processor + """ + self.params = params + self._msg_counter = 0 + + async def open_async(self, context): + """ + Called by processor host to initialize the event processor. + """ + logging.info("Connection established {}".format(context.partition_id)) + + async def close_async(self, context, reason): + """ + Called by processor host to indicate that the event processor is being stopped. + (Params) Context:Information about the partition + """ + logging.info("Connection closed (reason {}, id {}, offset {}, sq_number {})".format( + reason, context.partition_id, context.offset, context.sequence_number)) + + async def process_events_async(self, context, messages): + """ + Called by the processor host when a batch of events has arrived. + This is where the real work of the event processor is done. + (Params) Context: Information about the partition, Messages: The events to be processed. + """ + logging.info("Events processed {} {}".format(context.partition_id, messages)) + await context.checkpoint_async() + + async def process_error_async(self, context, error): + """ + Called when the underlying client experiences an error while receiving. + EventProcessorHost will take care of recovering from the error and + continuing to pump messages,so no action is required from + (Params) Context: Information about the partition, Error: The error that occured. + """ + logging.error("Event Processor Error {!r}".format(error)) \ No newline at end of file diff --git a/tests/test_checkpoint_manager.py b/tests/asynctests/test_checkpoint_manager.py similarity index 100% rename from tests/test_checkpoint_manager.py rename to tests/asynctests/test_checkpoint_manager.py diff --git a/tests/test_eh_partition_pump.py b/tests/asynctests/test_eh_partition_pump.py similarity index 100% rename from tests/test_eh_partition_pump.py rename to tests/asynctests/test_eh_partition_pump.py diff --git a/tests/test_iothub_receive_async.py b/tests/asynctests/test_iothub_receive_async.py similarity index 100% rename from tests/test_iothub_receive_async.py rename to tests/asynctests/test_iothub_receive_async.py diff --git a/tests/asynctests/test_longrunning_eph.py b/tests/asynctests/test_longrunning_eph.py new file mode 100644 index 0000000..2814563 --- /dev/null +++ b/tests/asynctests/test_longrunning_eph.py @@ -0,0 +1,154 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# ----------------------------------------------------------------------------------- + +import logging +import asyncio +import sys +import os +import signal +import functools +import time +import pytest + +from azure.eventprocessorhost import ( + AbstractEventProcessor, + AzureStorageCheckpointLeaseManager, + EventHubConfig, + EventProcessorHost, + EPHOptions) + +from azure.eventhub import EventHubClient, Sender, EventData +import sys +import logging +from logging.handlers import RotatingFileHandler + + +def get_logger(filename, level=logging.INFO): + azure_logger = logging.getLogger("azure") + azure_logger.setLevel(level) + uamqp_logger = logging.getLogger("uamqp") + uamqp_logger.setLevel(level) + + formatter = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s') + if filename: + file_handler = RotatingFileHandler(filename, maxBytes=20*1024*1024, backupCount=3) + file_handler.setFormatter(formatter) + azure_logger.addHandler(file_handler) + uamqp_logger.addHandler(file_handler) + + return azure_logger + +logger = get_logger("eph_test.log", logging.INFO) + + +class EventProcessor(AbstractEventProcessor): + """ + Example Implmentation of AbstractEventProcessor + """ + + def __init__(self, params=None): + """ + Init Event processor + """ + super().__init__(params) + self._msg_counter = 0 + + async def open_async(self, context): + """ + Called by processor host to initialize the event processor. + """ + logger.info("Connection established {}".format(context.partition_id)) + + async def close_async(self, context, reason): + """ + Called by processor host to indicate that the event processor is being stopped. + :param context: Information about the partition + :type context: ~azure.eventprocessorhost.PartitionContext + """ + logger.info("Connection closed (reason {}, id {}, offset {}, sq_number {})".format( + reason, + context.partition_id, + context.offset, + context.sequence_number)) + + async def process_events_async(self, context, messages): + """ + Called by the processor host when a batch of events has arrived. + This is where the real work of the event processor is done. + :param context: Information about the partition + :type context: ~azure.eventprocessorhost.PartitionContext + :param messages: The events to be processed. + :type messages: list[~azure.eventhub.common.EventData] + """ + logger.info("Processing id {}, offset {}, sq_number {})".format( + context.partition_id, + context.offset, + context.sequence_number)) + await context.checkpoint_async() + + async def process_error_async(self, context, error): + """ + Called when the underlying client experiences an error while receiving. + EventProcessorHost will take care of recovering from the error and + continuing to pump messages,so no action is required from + :param context: Information about the partition + :type context: ~azure.eventprocessorhost.PartitionContext + :param error: The error that occured. + """ + logger.info("Event Processor Error for partition {}, {!r}".format(context.partition_id, error)) + + +async def wait_and_close(host): + """ + Run EventProcessorHost for 30 then shutdown. + """ + await asyncio.sleep(30) + await host.close_async() + + +def test_long_running_eph(): + loop = asyncio.get_event_loop() + + # Storage Account Credentials + STORAGE_ACCOUNT_NAME = os.environ['AZURE_STORAGE_ACCOUNT'] + STORAGE_KEY = os.environ['AZURE_STORAGE_SAS_KEY'] + LEASE_CONTAINER_NAME = "testleases" + + NAMESPACE = os.environ['EVENT_HUB_NAMESPACE'] + EVENTHUB = os.environ['EVENT_HUB_NAME'] + USER = os.environ['EVENT_HUB_SAS_POLICY'] + KEY = os.environ['EVENT_HUB_SAS_KEY'] + + # Eventhub config and storage manager + eh_config = EventHubConfig(NAMESPACE, EVENTHUB, USER, KEY, consumer_group="$default") + eh_options = EPHOptions() + eh_options.release_pump_on_timeout = True + eh_options.debug_trace = False + eh_options.receive_timeout = 120 + storage_manager = AzureStorageCheckpointLeaseManager( + storage_account_name=STORAGE_ACCOUNT_NAME, + storage_account_key=STORAGE_KEY, + lease_renew_interval=30, + lease_container_name=LEASE_CONTAINER_NAME, + lease_duration=60) + + # Event loop and host + host = EventProcessorHost( + EventProcessor, + eh_config, + storage_manager, + ep_params=["param1","param2"], + eph_options=eh_options, + loop=loop) + + tasks = asyncio.gather( + host.open_async(), + wait_and_close(host), return_exceptions=True) + results = loop.run_until_complete(tasks) + assert not any(results) + + +if __name__ == '__main__': + test_long_running_eph() diff --git a/tests/asynctests/test_longrunning_receive_async.py b/tests/asynctests/test_longrunning_receive_async.py new file mode 100644 index 0000000..e846650 --- /dev/null +++ b/tests/asynctests/test_longrunning_receive_async.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python + +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +""" +receive test. +""" + +import logging +import asyncio +import argparse +import time +import os +import sys +from urllib.parse import quote_plus +from logging.handlers import RotatingFileHandler + +from azure.eventhub import Offset +from azure.eventhub import EventHubClientAsync + +def get_logger(filename, level=logging.INFO): + azure_logger = logging.getLogger("azure") + azure_logger.setLevel(level) + uamqp_logger = logging.getLogger("uamqp") + uamqp_logger.setLevel(level) + + formatter = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s') + if filename: + file_handler = RotatingFileHandler(filename, maxBytes=20*1024*1024, backupCount=3) + file_handler.setFormatter(formatter) + azure_logger.addHandler(file_handler) + uamqp_logger.addHandler(file_handler) + + return azure_logger + +logger = get_logger("recv_test_async.log", logging.INFO) + + +async def get_partitions(args): + eh_data = await args.get_eventhub_info_async() + return eh_data["partition_ids"] + + +async def pump(_pid, receiver, _args, _dl): + total = 0 + iteration = 0 + deadline = time.time() + _dl + try: + while time.time() < deadline: + batch = await receiver.receive(timeout=5) + size = len(batch) + total += size + iteration += 1 + if size == 0: + print("{}: No events received, queue size {}, delivered {}".format( + _pid, + receiver.queue_size, + total)) + elif iteration >= 50: + iteration = 0 + print("{}: total received {}, last sn={}, last offset={}".format( + _pid, + total, + batch[-1].sequence_number, + batch[-1].offset.value)) + print("{}: total received {}".format( + _pid, + total)) + except Exception as e: + print("Partition {} receiver failed: {}".format(_pid, e)) + raise + + +def test_long_running_receive_async(): + parser = argparse.ArgumentParser() + parser.add_argument("--duration", help="Duration in seconds of the test", type=int, default=30) + parser.add_argument("--consumer", help="Consumer group name", default="$default") + parser.add_argument("--partitions", help="Comma seperated partition IDs") + parser.add_argument("--offset", help="Starting offset", default="-1") + parser.add_argument("--conn-str", help="EventHub connection string", default=os.environ.get('EVENT_HUB_CONNECTION_STR')) + parser.add_argument("--eventhub", help="Name of EventHub") + parser.add_argument("--address", help="Address URI to the EventHub entity") + parser.add_argument("--sas-policy", help="Name of the shared access policy to authenticate with") + parser.add_argument("--sas-key", help="Shared access key") + + loop = asyncio.get_event_loop() + args, _ = parser.parse_known_args() + if args.conn_str: + client = EventHubClientAsync.from_connection_string( + args.conn_str, + eventhub=args.eventhub, debug=False) + elif args.address: + client = EventHubClientAsync( + args.address, + username=args.sas_policy, + password=args.sas_key) + else: + try: + import pytest + pytest.skip("Must specify either '--conn-str' or '--address'") + except ImportError: + raise ValueError("Must specify either '--conn-str' or '--address'") + + try: + if not args.partitions: + partitions = loop.run_until_complete(get_partitions(client)) + else: + partitions = args.partitions.split(",") + pumps = [] + for pid in partitions: + receiver = client.add_async_receiver( + consumer_group=args.consumer, + partition=pid, + offset=Offset(args.offset), + prefetch=50) + pumps.append(pump(pid, receiver, args, args.duration)) + loop.run_until_complete(client.run_async()) + loop.run_until_complete(asyncio.gather(*pumps)) + finally: + loop.run_until_complete(client.stop_async()) + + +if __name__ == '__main__': + test_long_running_receive_async() diff --git a/tests/test_longrunning_send_async.py b/tests/asynctests/test_longrunning_send_async.py similarity index 69% rename from tests/test_longrunning_send_async.py rename to tests/asynctests/test_longrunning_send_async.py index afc13fa..f1c765d 100644 --- a/tests/test_longrunning_send_async.py +++ b/tests/asynctests/test_longrunning_send_async.py @@ -13,12 +13,27 @@ from azure.eventhub import EventHubClientAsync, EventData -try: - import tests - logger = tests.get_logger("send_test.log", logging.INFO) -except ImportError: - logger = logging.getLogger("uamqp") - logger.setLevel(logging.INFO) +import sys +import logging +from logging.handlers import RotatingFileHandler + + +def get_logger(filename, level=logging.INFO): + azure_logger = logging.getLogger("azure") + azure_logger.setLevel(level) + uamqp_logger = logging.getLogger("uamqp") + uamqp_logger.setLevel(level) + + formatter = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s') + if filename: + file_handler = RotatingFileHandler(filename, maxBytes=20*1024*1024, backupCount=3) + file_handler.setFormatter(formatter) + azure_logger.addHandler(file_handler) + uamqp_logger.addHandler(file_handler) + + return azure_logger + +logger = get_logger("send_test_async.log", logging.INFO) def check_send_successful(outcome, condition): @@ -27,9 +42,6 @@ def check_send_successful(outcome, condition): async def get_partitions(args): - #client = EventHubClientAsync.from_connection_string( - # args.conn_str, - # eventhub=args.eventhub, debug=True) eh_data = await args.get_eventhub_info_async() return eh_data["partition_ids"] @@ -43,9 +55,9 @@ def data_generator(): yield b"D" * args.payload if args.batch > 1: - logger.error("Sending batched messages") + logger.info("{}: Sending batched messages".format(pid)) else: - logger.error("Sending single messages") + logger.info("{}: Sending single messages".format(pid)) try: while time.time() < deadline: @@ -55,12 +67,12 @@ def data_generator(): data = EventData(body=b"D" * args.payload) sender.transfer(data, callback=check_send_successful) total += args.batch - if total % 10000 == 0: + if total % 10 == 0: await sender.wait_async() - logger.error("Send total {}".format(total)) + logger.info("{}: Send total {}".format(pid, total)) except Exception as err: - logger.error("Send failed {}".format(err)) - logger.error("Sent total {}".format(total)) + logger.info("{}: Send failed {}".format(pid, err)) + logger.info("{}: Sent total {}".format(pid, total)) def test_long_running_partition_send_async(): @@ -82,10 +94,11 @@ def test_long_running_partition_send_async(): args.conn_str, eventhub=args.eventhub, debug=True) elif args.address: - client = EventHubClient( + client = EventHubClientAsync( args.address, username=args.sas_policy, - password=args.sas_key) + password=args.sas_key, + auth_timeout=500) else: try: import pytest @@ -100,10 +113,10 @@ def test_long_running_partition_send_async(): partitions = args.partitions.split(",") pumps = [] for pid in partitions: - sender = client.add_async_sender(partition=pid) + sender = client.add_async_sender(partition=pid, send_timeout=500) pumps.append(pump(pid, sender, args, args.duration)) loop.run_until_complete(client.run_async()) - loop.run_until_complete(asyncio.gather(*pumps)) + loop.run_until_complete(asyncio.gather(*pumps, return_exceptions=True)) finally: loop.run_until_complete(client.stop_async()) diff --git a/tests/asynctests/test_negative_async.py b/tests/asynctests/test_negative_async.py new file mode 100644 index 0000000..6204878 --- /dev/null +++ b/tests/asynctests/test_negative_async.py @@ -0,0 +1,185 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +#-------------------------------------------------------------------------- + +import os +import asyncio +import pytest +import time + +from azure import eventhub +from azure.eventhub import ( + EventHubClientAsync, + EventData, + Offset, + EventHubError) + + +@pytest.mark.asyncio +async def test_send_with_invalid_hostname_async(invalid_hostname, receivers): + client = EventHubClientAsync.from_connection_string(invalid_hostname, debug=True) + sender = client.add_async_sender() + with pytest.raises(EventHubError): + await client.run_async() + + +@pytest.mark.asyncio +async def test_receive_with_invalid_hostname_async(invalid_hostname): + client = EventHubClientAsync.from_connection_string(invalid_hostname, debug=True) + sender = client.add_async_receiver("$default", "0") + with pytest.raises(EventHubError): + await client.run_async() + + +@pytest.mark.asyncio +async def test_send_with_invalid_key_async(invalid_key, receivers): + client = EventHubClientAsync.from_connection_string(invalid_key, debug=False) + sender = client.add_async_sender() + with pytest.raises(EventHubError): + await client.run_async() + + +@pytest.mark.asyncio +async def test_receive_with_invalid_key_async(invalid_key): + client = EventHubClientAsync.from_connection_string(invalid_key, debug=True) + sender = client.add_async_receiver("$default", "0") + with pytest.raises(EventHubError): + await client.run_async() + + +@pytest.mark.asyncio +async def test_send_with_invalid_policy_async(invalid_policy, receivers): + client = EventHubClientAsync.from_connection_string(invalid_policy, debug=False) + sender = client.add_async_sender() + with pytest.raises(EventHubError): + await client.run_async() + + +@pytest.mark.asyncio +async def test_receive_with_invalid_policy_async(invalid_policy): + client = EventHubClientAsync.from_connection_string(invalid_policy, debug=True) + sender = client.add_async_receiver("$default", "0") + with pytest.raises(EventHubError): + await client.run_async() + + +@pytest.mark.asyncio +async def test_send_partition_key_with_partition_async(connection_str): + client = EventHubClientAsync.from_connection_string(connection_str, debug=True) + sender = client.add_async_sender(partition="1") + try: + await client.run_async() + data = EventData(b"Data") + data.partition_key = b"PKey" + with pytest.raises(ValueError): + await sender.send(data) + finally: + await client.stop_async() + + +@pytest.mark.asyncio +async def test_non_existing_entity_sender_async(connection_str): + client = EventHubClientAsync.from_connection_string(connection_str, eventhub="nemo", debug=False) + sender = client.add_async_sender(partition="1") + with pytest.raises(EventHubError): + await client.run_async() + + +@pytest.mark.asyncio +async def test_non_existing_entity_receiver_async(connection_str): + client = EventHubClientAsync.from_connection_string(connection_str, eventhub="nemo", debug=False) + receiver = client.add_async_receiver("$default", "0") + with pytest.raises(EventHubError): + await client.run_async() + + +@pytest.mark.asyncio +async def test_receive_from_invalid_partitions_async(connection_str): + partitions = ["XYZ", "-1", "1000", "-" ] + for p in partitions: + client = EventHubClientAsync.from_connection_string(connection_str, debug=True) + receiver = client.add_async_receiver("$default", p) + try: + with pytest.raises(EventHubError): + await client.run_async() + await receiver.receive(timeout=10) + finally: + await client.stop_async() + + +@pytest.mark.asyncio +async def test_send_to_invalid_partitions_async(connection_str): + partitions = ["XYZ", "-1", "1000", "-" ] + for p in partitions: + client = EventHubClientAsync.from_connection_string(connection_str, debug=False) + sender = client.add_async_sender(partition=p) + await client.run_async() + data = EventData(b"A" * 300000) + try: + with pytest.raises(EventHubError): + await sender.send(data) + finally: + await client.stop_async() + + +@pytest.mark.asyncio +async def test_send_too_large_message_async(connection_str): + client = EventHubClientAsync.from_connection_string(connection_str, debug=False) + sender = client.add_async_sender() + try: + await client.run_async() + data = EventData(b"A" * 300000) + with pytest.raises(EventHubError): + await sender.send(data) + finally: + await client.stop_async() + + +@pytest.mark.asyncio +async def test_send_null_body_async(connection_str): + client = EventHubClientAsync.from_connection_string(connection_str, debug=False) + sender = client.add_async_sender() + try: + await client.run_async() + with pytest.raises(ValueError): + data = EventData(None) + await sender.send(data) + finally: + await client.stop_async() + + +async def pump(receiver): + messages = 0 + count = 0 + batch = await receiver.receive(timeout=10) + while batch and count <= 5: + count += 1 + messages += len(batch) + batch = await receiver.receive(timeout=10) + return messages + + +@pytest.mark.asyncio +async def test_max_receivers_async(connection_str, senders): + client = EventHubClientAsync.from_connection_string(connection_str, debug=True) + receivers = [] + for i in range(6): + receivers.append(client.add_async_receiver("$default", "0", prefetch=1000, offset=Offset('@latest'))) + try: + await client.run_async() + outputs = await asyncio.gather( + pump(receivers[0]), + pump(receivers[1]), + pump(receivers[2]), + pump(receivers[3]), + pump(receivers[4]), + pump(receivers[5]), + return_exceptions=True) + print(outputs) + failed = [o for o in outputs if isinstance(o, EventHubError)] + assert len(failed) == 1 + print(failed[0].message) + finally: + await client.stop_async() diff --git a/tests/test_partition_manager.py b/tests/asynctests/test_partition_manager.py similarity index 100% rename from tests/test_partition_manager.py rename to tests/asynctests/test_partition_manager.py diff --git a/tests/test_partition_pump.py b/tests/asynctests/test_partition_pump.py similarity index 100% rename from tests/test_partition_pump.py rename to tests/asynctests/test_partition_pump.py diff --git a/tests/test_receive_async.py b/tests/asynctests/test_receive_async.py similarity index 100% rename from tests/test_receive_async.py rename to tests/asynctests/test_receive_async.py diff --git a/tests/asynctests/test_reconnect_async.py b/tests/asynctests/test_reconnect_async.py new file mode 100644 index 0000000..709ed49 --- /dev/null +++ b/tests/asynctests/test_reconnect_async.py @@ -0,0 +1,90 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +#-------------------------------------------------------------------------- + +import os +import time +import asyncio +import pytest + +from azure import eventhub +from azure.eventhub import ( + EventHubClientAsync, + EventData, + Offset, + EventHubError) + + +@pytest.mark.asyncio +async def test_send_with_long_interval_async(connection_str, receivers): + #pytest.skip("long running") + client = EventHubClientAsync.from_connection_string(connection_str, debug=True) + sender = client.add_async_sender() + try: + await client.run_async() + await sender.send(EventData(b"A single event")) + for _ in range(2): + await asyncio.sleep(300) + await sender.send(EventData(b"A single event")) + finally: + await client.stop_async() + + received = [] + for r in receivers: + received.extend(r.receive(timeout=1)) + assert len(received) == 3 + assert list(received[0].body)[0] == b"A single event" + + +def pump(receiver): + messages = [] + batch = receiver.receive(timeout=1) + messages.extend(batch) + while batch: + batch = receiver.receive(timeout=1) + messages.extend(batch) + return messages + +@pytest.mark.asyncio +async def test_send_with_forced_conn_close_async(connection_str, receivers): + #pytest.skip("long running") + client = EventHubClientAsync.from_connection_string(connection_str, debug=True) + sender = client.add_async_sender() + try: + await client.run_async() + await sender.send(EventData(b"A single event")) + sender._handler._message_sender.destroy() + await asyncio.sleep(300) + await sender.send(EventData(b"A single event")) + await sender.send(EventData(b"A single event")) + sender._handler._message_sender.destroy() + await asyncio.sleep(300) + await sender.send(EventData(b"A single event")) + await sender.send(EventData(b"A single event")) + finally: + await client.stop_async() + + received = [] + for r in receivers: + received.extend(pump(r)) + assert len(received) == 5 + assert list(received[0].body)[0] == b"A single event" + + +# def test_send_with_forced_link_detach(connection_str, receivers): +# client = EventHubClient.from_connection_string(connection_str, debug=True) +# sender = client.add_sender() +# size = 20 * 1024 +# try: +# client.run() +# for i in range(1000): +# sender.transfer(EventData([b"A"*size, b"B"*size, b"C"*size, b"D"*size, b"A"*size, b"B"*size, b"C"*size, b"D"*size, b"A"*size, b"B"*size, b"C"*size, b"D"*size])) +# sender.wait() +# finally: +# client.stop() + +# received = [] +# for r in receivers: +# received.extend(r.receive(timeout=10)) diff --git a/tests/test_send_async.py b/tests/asynctests/test_send_async.py similarity index 100% rename from tests/test_send_async.py rename to tests/asynctests/test_send_async.py diff --git a/tests/test_longrunning_receive.py b/tests/test_longrunning_receive.py index b32731b..171d7cf 100644 --- a/tests/test_longrunning_receive.py +++ b/tests/test_longrunning_receive.py @@ -10,50 +10,65 @@ """ import logging -import asyncio import argparse import time import os -from urllib.parse import quote_plus +import sys + +from logging.handlers import RotatingFileHandler from azure.eventhub import Offset -from azure.eventhub import EventHubClientAsync +from azure.eventhub import EventHubClient + +def get_logger(filename, level=logging.INFO): + azure_logger = logging.getLogger("azure") + azure_logger.setLevel(level) + uamqp_logger = logging.getLogger("uamqp") + uamqp_logger.setLevel(level) + + formatter = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s') + if filename: + file_handler = RotatingFileHandler(filename, maxBytes=20*1024*1024, backupCount=3) + file_handler.setFormatter(formatter) + azure_logger.addHandler(file_handler) + uamqp_logger.addHandler(file_handler) + + return azure_logger + +logger = get_logger("recv_test.log", logging.INFO) + -try: - import tests - logger = tests.get_logger("recv_test.log", logging.INFO) -except ImportError: - logger = logging.getLogger("uamqp") - logger.setLevel(logging.INFO) +def get_partitions(args): + eh_data = args.get_eventhub_info() + return eh_data["partition_ids"] -async def pump(_pid, receiver, _args, _dl): +def pump(receivers, duration): total = 0 iteration = 0 - deadline = time.time() + _dl + deadline = time.time() + duration try: while time.time() < deadline: - batch = await receiver.receive(timeout=5) - size = len(batch) - total += size - iteration += 1 - if size == 0: - print("{}: No events received, queue size {}, delivered {}".format( - _pid, - receiver.queue_size, - total)) - elif iteration >= 80: - iteration = 0 - print("{}: total received {}, last sn={}, last offset={}".format( - _pid, - total, - batch[-1].sequence_number, - batch[-1].offset.value)) - print("{}: total received {}".format( - _pid, - total)) + for pid, receiver in receivers.items(): + batch = receiver.receive(timeout=5) + size = len(batch) + total += size + iteration += 1 + if size == 0: + print("{}: No events received, queue size {}, delivered {}".format( + pid, + receiver.queue_size, + total)) + elif iteration >= 50: + iteration = 0 + print("{}: total received {}, last sn={}, last offset={}".format( + pid, + total, + batch[-1].sequence_number, + batch[-1].offset.value)) + print("Total received {}".format(total)) except Exception as e: - print("Partition {} receiver failed: {}".format(_pid, e)) + print("Receiver failed: {}".format(e)) raise @@ -61,7 +76,7 @@ def test_long_running_receive(): parser = argparse.ArgumentParser() parser.add_argument("--duration", help="Duration in seconds of the test", type=int, default=30) parser.add_argument("--consumer", help="Consumer group name", default="$default") - parser.add_argument("--partitions", help="Comma seperated partition IDs", default="0") + parser.add_argument("--partitions", help="Comma seperated partition IDs") parser.add_argument("--offset", help="Starting offset", default="-1") parser.add_argument("--conn-str", help="EventHub connection string", default=os.environ.get('EVENT_HUB_CONNECTION_STR')) parser.add_argument("--eventhub", help="Name of EventHub") @@ -69,14 +84,13 @@ def test_long_running_receive(): parser.add_argument("--sas-policy", help="Name of the shared access policy to authenticate with") parser.add_argument("--sas-key", help="Shared access key") - loop = asyncio.get_event_loop() args, _ = parser.parse_known_args() if args.conn_str: - client = EventHubClientAsync.from_connection_string( + client = EventHubClient.from_connection_string( args.conn_str, eventhub=args.eventhub, debug=False) elif args.address: - client = EventHubClientAsync( + client = EventHubClient( args.address, username=args.sas_policy, password=args.sas_key) @@ -88,18 +102,21 @@ def test_long_running_receive(): raise ValueError("Must specify either '--conn-str' or '--address'") try: - pumps = [] - for pid in args.partitions.split(","): - receiver = client.add_async_receiver( + if not args.partitions: + partitions = get_partitions(client) + else: + partitions = args.partitions.split(",") + pumps = {} + for pid in partitions: + pumps[pid] = client.add_receiver( consumer_group=args.consumer, partition=pid, offset=Offset(args.offset), - prefetch=5000) - pumps.append(pump(pid, receiver, args, args.duration)) - loop.run_until_complete(client.run_async()) - loop.run_until_complete(asyncio.gather(*pumps)) + prefetch=50) + client.run() + pump(pumps, args.duration) finally: - loop.run_until_complete(client.stop_async()) + client.stop() if __name__ == '__main__': diff --git a/tests/test_longrunning_send.py b/tests/test_longrunning_send.py index 29957e4..776488c 100644 --- a/tests/test_longrunning_send.py +++ b/tests/test_longrunning_send.py @@ -12,12 +12,27 @@ from azure.eventhub import EventHubClient, Sender, EventData -try: - import tests - logger = tests.get_logger("send_test.log", logging.INFO) -except ImportError: - logger = logging.getLogger("uamqp") - logger.setLevel(logging.INFO) +import sys +import logging +from logging.handlers import RotatingFileHandler + + +def get_logger(filename, level=logging.INFO): + azure_logger = logging.getLogger("azure") + azure_logger.setLevel(level) + uamqp_logger = logging.getLogger("uamqp") + uamqp_logger.setLevel(level) + + formatter = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s') + if filename: + file_handler = RotatingFileHandler(filename, maxBytes=20*1024*1024, backupCount=3) + file_handler.setFormatter(formatter) + azure_logger.addHandler(file_handler) + uamqp_logger.addHandler(file_handler) + + return azure_logger + +logger = get_logger("send_test.log", logging.INFO) def check_send_successful(outcome, condition): diff --git a/tests/test_negative.py b/tests/test_negative.py index bdfcbfd..f7724fb 100644 --- a/tests/test_negative.py +++ b/tests/test_negative.py @@ -5,13 +5,11 @@ #-------------------------------------------------------------------------- import os -import asyncio import pytest import time from azure import eventhub from azure.eventhub import ( - EventHubClientAsync, EventData, Offset, EventHubError, @@ -25,14 +23,6 @@ def test_send_with_invalid_hostname(invalid_hostname, receivers): client.run() -@pytest.mark.asyncio -async def test_send_with_invalid_hostname_async(invalid_hostname, receivers): - client = EventHubClientAsync.from_connection_string(invalid_hostname, debug=True) - sender = client.add_async_sender() - with pytest.raises(EventHubError): - await client.run_async() - - def test_receive_with_invalid_hostname_sync(invalid_hostname): client = EventHubClient.from_connection_string(invalid_hostname, debug=True) receiver = client.add_receiver("$default", "0") @@ -40,14 +30,6 @@ def test_receive_with_invalid_hostname_sync(invalid_hostname): client.run() -@pytest.mark.asyncio -async def test_receive_with_invalid_hostname_async(invalid_hostname): - client = EventHubClientAsync.from_connection_string(invalid_hostname, debug=True) - sender = client.add_async_receiver("$default", "0") - with pytest.raises(EventHubError): - await client.run_async() - - def test_send_with_invalid_key(invalid_key, receivers): client = EventHubClient.from_connection_string(invalid_key, debug=False) sender = client.add_sender() @@ -55,14 +37,6 @@ def test_send_with_invalid_key(invalid_key, receivers): client.run() -@pytest.mark.asyncio -async def test_send_with_invalid_key_async(invalid_key, receivers): - client = EventHubClientAsync.from_connection_string(invalid_key, debug=False) - sender = client.add_async_sender() - with pytest.raises(EventHubError): - await client.run_async() - - def test_receive_with_invalid_key_sync(invalid_key): client = EventHubClient.from_connection_string(invalid_key, debug=True) receiver = client.add_receiver("$default", "0") @@ -70,14 +44,6 @@ def test_receive_with_invalid_key_sync(invalid_key): client.run() -@pytest.mark.asyncio -async def test_receive_with_invalid_key_async(invalid_key): - client = EventHubClientAsync.from_connection_string(invalid_key, debug=True) - sender = client.add_async_receiver("$default", "0") - with pytest.raises(EventHubError): - await client.run_async() - - def test_send_with_invalid_policy(invalid_policy, receivers): client = EventHubClient.from_connection_string(invalid_policy, debug=False) sender = client.add_sender() @@ -85,14 +51,6 @@ def test_send_with_invalid_policy(invalid_policy, receivers): client.run() -@pytest.mark.asyncio -async def test_send_with_invalid_policy_async(invalid_policy, receivers): - client = EventHubClientAsync.from_connection_string(invalid_policy, debug=False) - sender = client.add_async_sender() - with pytest.raises(EventHubError): - await client.run_async() - - def test_receive_with_invalid_policy_sync(invalid_policy): client = EventHubClient.from_connection_string(invalid_policy, debug=True) receiver = client.add_receiver("$default", "0") @@ -100,14 +58,6 @@ def test_receive_with_invalid_policy_sync(invalid_policy): client.run() -@pytest.mark.asyncio -async def test_receive_with_invalid_policy_async(invalid_policy): - client = EventHubClientAsync.from_connection_string(invalid_policy, debug=True) - sender = client.add_async_receiver("$default", "0") - with pytest.raises(EventHubError): - await client.run_async() - - def test_send_partition_key_with_partition_sync(connection_str): client = EventHubClient.from_connection_string(connection_str, debug=True) sender = client.add_sender(partition="1") @@ -121,20 +71,6 @@ def test_send_partition_key_with_partition_sync(connection_str): client.stop() -@pytest.mark.asyncio -async def test_send_partition_key_with_partition_async(connection_str): - client = EventHubClientAsync.from_connection_string(connection_str, debug=True) - sender = client.add_async_sender(partition="1") - try: - await client.run_async() - data = EventData(b"Data") - data.partition_key = b"PKey" - with pytest.raises(ValueError): - await sender.send(data) - finally: - await client.stop_async() - - def test_non_existing_entity_sender(connection_str): client = EventHubClient.from_connection_string(connection_str, eventhub="nemo", debug=False) sender = client.add_sender(partition="1") @@ -142,14 +78,6 @@ def test_non_existing_entity_sender(connection_str): client.run() -@pytest.mark.asyncio -async def test_non_existing_entity_sender_async(connection_str): - client = EventHubClientAsync.from_connection_string(connection_str, eventhub="nemo", debug=False) - sender = client.add_async_sender(partition="1") - with pytest.raises(EventHubError): - await client.run_async() - - def test_non_existing_entity_receiver(connection_str): client = EventHubClient.from_connection_string(connection_str, eventhub="nemo", debug=False) receiver = client.add_receiver("$default", "0") @@ -157,14 +85,6 @@ def test_non_existing_entity_receiver(connection_str): client.run() -@pytest.mark.asyncio -async def test_non_existing_entity_receiver_async(connection_str): - client = EventHubClientAsync.from_connection_string(connection_str, eventhub="nemo", debug=False) - receiver = client.add_async_receiver("$default", "0") - with pytest.raises(EventHubError): - await client.run_async() - - def test_receive_from_invalid_partitions_sync(connection_str): partitions = ["XYZ", "-1", "1000", "-" ] for p in partitions: @@ -178,20 +98,6 @@ def test_receive_from_invalid_partitions_sync(connection_str): client.stop() -@pytest.mark.asyncio -async def test_receive_from_invalid_partitions_async(connection_str): - partitions = ["XYZ", "-1", "1000", "-" ] - for p in partitions: - client = EventHubClientAsync.from_connection_string(connection_str, debug=True) - receiver = client.add_async_receiver("$default", p) - try: - with pytest.raises(EventHubError): - await client.run_async() - await receiver.receive(timeout=10) - finally: - await client.stop_async() - - def test_send_to_invalid_partitions(connection_str): partitions = ["XYZ", "-1", "1000", "-" ] for p in partitions: @@ -206,21 +112,6 @@ def test_send_to_invalid_partitions(connection_str): client.stop() -@pytest.mark.asyncio -async def test_send_to_invalid_partitions_async(connection_str): - partitions = ["XYZ", "-1", "1000", "-" ] - for p in partitions: - client = EventHubClientAsync.from_connection_string(connection_str, debug=False) - sender = client.add_async_sender(partition=p) - await client.run_async() - data = EventData(b"A" * 300000) - try: - with pytest.raises(EventHubError): - await sender.send(data) - finally: - await client.stop_async() - - def test_send_too_large_message(connection_str): client = EventHubClient.from_connection_string(connection_str, debug=True) sender = client.add_sender() @@ -233,19 +124,6 @@ def test_send_too_large_message(connection_str): client.stop() -@pytest.mark.asyncio -async def test_send_too_large_message_async(connection_str): - client = EventHubClientAsync.from_connection_string(connection_str, debug=False) - sender = client.add_async_sender() - try: - await client.run_async() - data = EventData(b"A" * 300000) - with pytest.raises(EventHubError): - await sender.send(data) - finally: - await client.stop_async() - - def test_send_null_body(connection_str): partitions = ["XYZ", "-1", "1000", "-" ] client = EventHubClient.from_connection_string(connection_str, debug=False) @@ -259,54 +137,6 @@ def test_send_null_body(connection_str): client.stop() -@pytest.mark.asyncio -async def test_send_null_body_async(connection_str): - client = EventHubClientAsync.from_connection_string(connection_str, debug=False) - sender = client.add_async_sender() - try: - await client.run_async() - with pytest.raises(ValueError): - data = EventData(None) - await sender.send(data) - finally: - await client.stop_async() - - -async def pump(receiver): - messages = 0 - count = 0 - batch = await receiver.receive(timeout=10) - while batch and count <= 5: - count += 1 - messages += len(batch) - batch = await receiver.receive(timeout=10) - return messages - - -@pytest.mark.asyncio -async def test_max_receivers_async(connection_str, senders): - client = EventHubClientAsync.from_connection_string(connection_str, debug=True) - receivers = [] - for i in range(6): - receivers.append(client.add_async_receiver("$default", "0", prefetch=1000, offset=Offset('@latest'))) - try: - await client.run_async() - outputs = await asyncio.gather( - pump(receivers[0]), - pump(receivers[1]), - pump(receivers[2]), - pump(receivers[3]), - pump(receivers[4]), - pump(receivers[5]), - return_exceptions=True) - print(outputs) - failed = [o for o in outputs if isinstance(o, EventHubError)] - assert len(failed) == 1 - print(failed[0].message) - finally: - await client.stop_async() - - def test_message_body_types(connection_str, senders): client = EventHubClient.from_connection_string(connection_str, debug=False) receiver = client.add_receiver("$default", "0", offset=Offset('@latest')) diff --git a/tests/test_reconnect.py b/tests/test_reconnect.py index bd10bbd..645fa96 100644 --- a/tests/test_reconnect.py +++ b/tests/test_reconnect.py @@ -6,12 +6,10 @@ import os import time -import asyncio import pytest from azure import eventhub from azure.eventhub import ( - EventHubClientAsync, EventData, Offset, EventHubError, @@ -39,27 +37,6 @@ def test_send_with_long_interval_sync(connection_str, receivers): assert list(received[0].body)[0] == b"A single event" -@pytest.mark.asyncio -async def test_send_with_long_interval_async(connection_str, receivers): - #pytest.skip("long running") - client = EventHubClientAsync.from_connection_string(connection_str, debug=True) - sender = client.add_async_sender() - try: - await client.run_async() - await sender.send(EventData(b"A single event")) - for _ in range(2): - await asyncio.sleep(300) - await sender.send(EventData(b"A single event")) - finally: - await client.stop_async() - - received = [] - for r in receivers: - received.extend(r.receive(timeout=1)) - assert len(received) == 3 - assert list(received[0].body)[0] == b"A single event" - - def test_send_with_forced_conn_close_sync(connection_str, receivers): #pytest.skip("long running") client = EventHubClient.from_connection_string(connection_str, debug=True) @@ -85,41 +62,6 @@ def test_send_with_forced_conn_close_sync(connection_str, receivers): assert list(received[0].body)[0] == b"A single event" -def pump(receiver): - messages = [] - batch = receiver.receive(timeout=1) - messages.extend(batch) - while batch: - batch = receiver.receive(timeout=1) - messages.extend(batch) - return messages - -@pytest.mark.asyncio -async def test_send_with_forced_conn_close_async(connection_str, receivers): - #pytest.skip("long running") - client = EventHubClientAsync.from_connection_string(connection_str, debug=True) - sender = client.add_async_sender() - try: - await client.run_async() - await sender.send(EventData(b"A single event")) - sender._handler._message_sender.destroy() - await asyncio.sleep(300) - await sender.send(EventData(b"A single event")) - await sender.send(EventData(b"A single event")) - sender._handler._message_sender.destroy() - await asyncio.sleep(300) - await sender.send(EventData(b"A single event")) - await sender.send(EventData(b"A single event")) - finally: - await client.stop_async() - - received = [] - for r in receivers: - received.extend(pump(r)) - assert len(received) == 5 - assert list(received[0].body)[0] == b"A single event" - - # def test_send_with_forced_link_detach(connection_str, receivers): # client = EventHubClient.from_connection_string(connection_str, debug=True) # sender = client.add_sender() From 6acd0df452a261d72aede4b3d5eda6edd9d07c38 Mon Sep 17 00:00:00 2001 From: annatisch Date: Mon, 1 Oct 2018 14:49:48 -0700 Subject: [PATCH 48/52] Support 2.7 types --- azure/eventhub/async_ops/__init__.py | 5 +---- azure/eventhub/client.py | 5 ++++- azure/eventhub/common.py | 17 +++++++++++------ azure/eventhub/receiver.py | 3 ++- azure/eventhub/sender.py | 1 + 5 files changed, 19 insertions(+), 12 deletions(-) diff --git a/azure/eventhub/async_ops/__init__.py b/azure/eventhub/async_ops/__init__.py index 7c62b3d..375cf3b 100644 --- a/azure/eventhub/async_ops/__init__.py +++ b/azure/eventhub/async_ops/__init__.py @@ -7,10 +7,7 @@ import asyncio import time import datetime -try: - from urllib import urlparse, unquote_plus, urlencode, quote_plus -except ImportError: - from urllib.parse import urlparse, unquote_plus, urlencode, quote_plus +from urllib.parse import urlparse, unquote_plus, urlencode, quote_plus from uamqp import authentication, constants, types, errors from uamqp import ( diff --git a/azure/eventhub/client.py b/azure/eventhub/client.py index 06508df..4e0cc7a 100644 --- a/azure/eventhub/client.py +++ b/azure/eventhub/client.py @@ -2,6 +2,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- +from __future__ import unicode_literals import logging import datetime @@ -10,10 +11,12 @@ import time import functools try: - from urllib import urlparse, unquote_plus, urlencode, quote_plus + from urlparse import urlparse + from urllib import unquote_plus, urlencode, quote_plus except ImportError: from urllib.parse import urlparse, unquote_plus, urlencode, quote_plus +import six import uamqp from uamqp import Message from uamqp import authentication diff --git a/azure/eventhub/common.py b/azure/eventhub/common.py index b4a1755..28745b4 100644 --- a/azure/eventhub/common.py +++ b/azure/eventhub/common.py @@ -2,11 +2,14 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- +from __future__ import unicode_literals import datetime import time import json +import six + from uamqp import Message, BatchMessage from uamqp import types, constants, errors from uamqp.message import MessageHeader, MessageProperties @@ -63,6 +66,8 @@ def __init__(self, body=None, batch=None, to_device=None, message=None): :type body: str, bytes or list :param batch: A data generator to send batched messages. :type batch: Generator + :param to_device: An IoT device to route to. + :type to_device: str :param message: The received message. :type message: ~uamqp.message.Message """ @@ -94,7 +99,7 @@ def sequence_number(self): """ The sequence number of the event data object. - :rtype: int + :rtype: int or long """ return self._annotations.get(EventData.PROP_SEQ_NUMBER, None) @@ -103,7 +108,7 @@ def offset(self): """ The offset of the event data object. - :rtype: int + :rtype: ~azure.eventhub.common.Offset """ try: return Offset(self._annotations[EventData.PROP_OFFSET].decode('UTF-8')) @@ -200,13 +205,13 @@ def body_as_str(self, encoding='UTF-8'): :param encoding: The encoding to use for decoding message data. Default is 'UTF-8' - :rtype: str + :rtype: str or unicode """ data = self.body try: return "".join(b.decode(encoding) for b in data) except TypeError: - return str(data) + return six.text_type(data) except: # pylint: disable=bare-except pass try: @@ -269,7 +274,7 @@ def selector(self): if isinstance(self.value, datetime.datetime): timestamp = (time.mktime(self.value.timetuple()) * 1000) + (self.value.microsecond/1000) return ("amqp.annotation.x-opt-enqueued-time {} '{}'".format(operator, int(timestamp))).encode('utf-8') - if isinstance(self.value, int): + if isinstance(self.value, six.integer_types): return ("amqp.annotation.x-opt-sequence-number {} '{}'".format(operator, self.value)).encode('utf-8') return ("amqp.annotation.x-opt-offset {} '{}'".format(operator, self.value)).encode('utf-8') @@ -310,7 +315,7 @@ def __init__(self, message, details=None): def _parse_error(self, error_list): details = [] - self.message = error_list if isinstance(error_list, str) else error_list.decode('UTF-8') + self.message = error_list if isinstance(error_list, six.text_types) else error_list.decode('UTF-8') details_index = self.message.find(" Reference:") if details_index >= 0: details_msg = self.message[details_index + 1:] diff --git a/azure/eventhub/receiver.py b/azure/eventhub/receiver.py index 6822149..0af8f3b 100644 --- a/azure/eventhub/receiver.py +++ b/azure/eventhub/receiver.py @@ -2,6 +2,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- +from __future__ import unicode_literals import uuid import logging @@ -106,7 +107,7 @@ def reconnect(self): # pylint: disable=protected-access alt_creds = { "username": self.client._auth_config.get("iot_username"), - "password":self.client._auth_config.get("iot_password")} + "password": self.client._auth_config.get("iot_password")} self._handler.close() source = Source(self.source) if self.offset is not None: diff --git a/azure/eventhub/sender.py b/azure/eventhub/sender.py index b4ed3b7..62fd71c 100644 --- a/azure/eventhub/sender.py +++ b/azure/eventhub/sender.py @@ -2,6 +2,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- +from __future__ import unicode_literals import uuid import logging From 584c0a3ed3d64c7362947fde77f9e7c770b3304b Mon Sep 17 00:00:00 2001 From: annatisch Date: Mon, 1 Oct 2018 14:50:09 -0700 Subject: [PATCH 49/52] Bumped version --- HISTORY.rst | 2 +- azure/__init__.py | 2 +- azure/eventhub/__init__.py | 2 +- setup.py | 10 +++++++--- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 56b246f..c9e5459 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,7 +6,7 @@ Release History 1.2.0 (release-candidate) +++++++++++++++++++++++++ -- Support for Python 2.7. +- Support for Python 2.7 in azure.eventhub module (azure.eventprocessorhost will not support Python 2.7). 1.1.0 (2018-09-21) diff --git a/azure/__init__.py b/azure/__init__.py index ef7c7c3..ef12b30 100644 --- a/azure/__init__.py +++ b/azure/__init__.py @@ -1,2 +1,2 @@ -__import__('pkg_resources').declare_namespace(__name__) \ No newline at end of file +__path__ = __import__('pkgutil').extend_path(__path__, __name__) \ No newline at end of file diff --git a/azure/eventhub/__init__.py b/azure/eventhub/__init__.py index e07a603..2d26557 100644 --- a/azure/eventhub/__init__.py +++ b/azure/eventhub/__init__.py @@ -3,7 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -__version__ = "1.1.0" +__version__ = "1.2.0rc1" from azure.eventhub.common import EventData, EventHubError, Offset from azure.eventhub.client import EventHubClient diff --git a/setup.py b/setup.py index cbb4d25..f597f09 100644 --- a/setup.py +++ b/setup.py @@ -56,10 +56,14 @@ 'License :: OSI Approved :: MIT License', ], zip_safe=False, - packages=find_packages(exclude=["examples", "tests"]), + packages=find_packages(exclude=[ + "azure", + "examples", + "tests", + "tests.asynctests"]), install_requires=[ - 'uamqp>=1.0.0,<2.0.0', - 'msrestazure~=0.5', + 'uamqp>=1.1.0rc1,<2.0.0', + 'msrestazure>=0.4.32,<2.0.0', 'azure-common~=1.1', 'azure-storage-blob~=1.3' ] From 893add6cb2df49837ee499c6cb662ea6b68eae95 Mon Sep 17 00:00:00 2001 From: annatisch Date: Mon, 1 Oct 2018 15:31:37 -0700 Subject: [PATCH 50/52] Added non-ascii tests --- tests/asynctests/test_send_async.py | 21 +++++++++++++++++++++ tests/test_send.py | 20 ++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/tests/asynctests/test_send_async.py b/tests/asynctests/test_send_async.py index ede8b73..c9173e8 100644 --- a/tests/asynctests/test_send_async.py +++ b/tests/asynctests/test_send_async.py @@ -1,3 +1,4 @@ +# -- coding: utf-8 -- #------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for @@ -8,6 +9,7 @@ import asyncio import pytest import time +import json from azure.eventhub import EventData, EventHubClientAsync @@ -123,6 +125,25 @@ async def test_send_partition_async(connection_str, receivers): assert len(partition_1) == 1 +@pytest.mark.asyncio +async def test_send_non_ascii_async(connection_str, receivers): + client = EventHubClientAsync.from_connection_string(connection_str, debug=False) + sender = client.add_async_sender(partition="0") + try: + await client.run_async() + await sender.send(EventData("é,è,à,ù,â,ê,î,ô,û")) + await sender.send(EventData(json.dumps({"foo": "漢字"}))) + except: + raise + finally: + await client.stop_async() + + partition_0 = receivers[0].receive(timeout=2) + assert len(partition_0) == 2 + assert partition_0[0].body_as_str() == "é,è,à,ù,â,ê,î,ô,û" + assert partition_0[1].body_as_json() == {"foo": "漢字"} + + @pytest.mark.asyncio async def test_send_partition_batch_async(connection_str, receivers): def batched(): diff --git a/tests/test_send.py b/tests/test_send.py index a74c3d1..6a88ee1 100644 --- a/tests/test_send.py +++ b/tests/test_send.py @@ -1,3 +1,4 @@ +# -- coding: utf-8 -- #------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for @@ -7,6 +8,7 @@ import os import pytest import time +import json from azure import eventhub from azure.eventhub import EventData, EventHubClient @@ -142,6 +144,24 @@ def test_send_partition(connection_str, receivers): assert len(partition_1) == 1 +def test_send_non_ascii(connection_str, receivers): + client = EventHubClient.from_connection_string(connection_str, debug=False) + sender = client.add_sender(partition="0") + try: + client.run() + sender.send(EventData(u"é,è,à,ù,â,ê,î,ô,û")) + sender.send(EventData(json.dumps({"foo": u"漢字"}))) + except: + raise + finally: + client.stop() + + partition_0 = receivers[0].receive(timeout=2) + assert len(partition_0) == 2 + assert partition_0[0].body_as_str() == u"é,è,à,ù,â,ê,î,ô,û" + assert partition_0[1].body_as_json() == {"foo": u"漢字"} + + def test_send_partition_batch(connection_str, receivers): def batched(): for i in range(10): From c7128cd9087bd77d761c2fb3b5062099e2f134ea Mon Sep 17 00:00:00 2001 From: annatisch Date: Mon, 1 Oct 2018 15:37:59 -0700 Subject: [PATCH 51/52] Fix CI --- azure/eventhub/client.py | 1 - azure/eventhub/common.py | 2 +- tests/asynctests/test_longrunning_eph.py | 19 +++++++++++-------- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/azure/eventhub/client.py b/azure/eventhub/client.py index 4e0cc7a..2e95717 100644 --- a/azure/eventhub/client.py +++ b/azure/eventhub/client.py @@ -16,7 +16,6 @@ except ImportError: from urllib.parse import urlparse, unquote_plus, urlencode, quote_plus -import six import uamqp from uamqp import Message from uamqp import authentication diff --git a/azure/eventhub/common.py b/azure/eventhub/common.py index 28745b4..1f9cd86 100644 --- a/azure/eventhub/common.py +++ b/azure/eventhub/common.py @@ -315,7 +315,7 @@ def __init__(self, message, details=None): def _parse_error(self, error_list): details = [] - self.message = error_list if isinstance(error_list, six.text_types) else error_list.decode('UTF-8') + self.message = error_list if isinstance(error_list, six.text_type) else error_list.decode('UTF-8') details_index = self.message.find(" Reference:") if details_index >= 0: details_msg = self.message[details_index + 1:] diff --git a/tests/asynctests/test_longrunning_eph.py b/tests/asynctests/test_longrunning_eph.py index 2814563..8170e3f 100644 --- a/tests/asynctests/test_longrunning_eph.py +++ b/tests/asynctests/test_longrunning_eph.py @@ -112,14 +112,17 @@ def test_long_running_eph(): loop = asyncio.get_event_loop() # Storage Account Credentials - STORAGE_ACCOUNT_NAME = os.environ['AZURE_STORAGE_ACCOUNT'] - STORAGE_KEY = os.environ['AZURE_STORAGE_SAS_KEY'] - LEASE_CONTAINER_NAME = "testleases" - - NAMESPACE = os.environ['EVENT_HUB_NAMESPACE'] - EVENTHUB = os.environ['EVENT_HUB_NAME'] - USER = os.environ['EVENT_HUB_SAS_POLICY'] - KEY = os.environ['EVENT_HUB_SAS_KEY'] + try: + STORAGE_ACCOUNT_NAME = os.environ['AZURE_STORAGE_ACCOUNT'] + STORAGE_KEY = os.environ['AZURE_STORAGE_SAS_KEY'] + LEASE_CONTAINER_NAME = "testleases" + + NAMESPACE = os.environ['EVENT_HUB_NAMESPACE'] + EVENTHUB = os.environ['EVENT_HUB_NAME'] + USER = os.environ['EVENT_HUB_SAS_POLICY'] + KEY = os.environ['EVENT_HUB_SAS_KEY'] + except KeyError: + pytest.skip("Missing live configuration.") # Eventhub config and storage manager eh_config = EventHubConfig(NAMESPACE, EVENTHUB, USER, KEY, consumer_group="$default") From 2a97956187c183b787dafe4a7e5fb0e3551eb6d8 Mon Sep 17 00:00:00 2001 From: annatisch Date: Mon, 1 Oct 2018 15:50:08 -0700 Subject: [PATCH 52/52] Fix Py27 pylint --- azure/eventhub/receiver.py | 2 +- azure/eventhub/sender.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/azure/eventhub/receiver.py b/azure/eventhub/receiver.py index 0af8f3b..5fe7035 100644 --- a/azure/eventhub/receiver.py +++ b/azure/eventhub/receiver.py @@ -15,7 +15,7 @@ log = logging.getLogger(__name__) -class Receiver: +class Receiver(object): """ Implements a Receiver. """ diff --git a/azure/eventhub/sender.py b/azure/eventhub/sender.py index 62fd71c..247dc80 100644 --- a/azure/eventhub/sender.py +++ b/azure/eventhub/sender.py @@ -15,7 +15,7 @@ log = logging.getLogger(__name__) -class Sender: +class Sender(object): """ Implements a Sender. """