Skip to content

Commit

Permalink
fix: retry and timeout values do not propagate in requests during pag…
Browse files Browse the repository at this point in the history
…ination (#2065)
  • Loading branch information
parthea authored Jul 23, 2024
1 parent 8c517a0 commit 76aa98e
Show file tree
Hide file tree
Showing 40 changed files with 665 additions and 83 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -509,6 +509,8 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta):
method=rpc,
request=request,
response=response,
retry=retry,
timeout=timeout,
metadata=metadata,
)
{% endif %}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,13 @@
{# This lives within the loop in order to ensure that this template
is empty if there are no paged methods.
-#}
from typing import Any, Callable, Iterable, Sequence, Tuple, Optional, Iterator
from google.api_core import gapic_v1
from google.api_core import retry as retries
from typing import Any, Callable, Iterable, Sequence, Tuple, Optional, Iterator, Union
try:
OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault, None]
except AttributeError: # pragma: NO COVER
OptionalRetry = Union[retries.Retry, object, None] # type: ignore

{% filter sort_lines %}
{% for method in service.methods.values() | selectattr('paged_result_field') %}
Expand Down Expand Up @@ -41,6 +47,8 @@ class {{ method.name }}Pager:
method: Callable[..., {{ method.output.ident }}],
request: {{ method.input.ident }},
response: {{ method.output.ident }},
retry: OptionalRetry = gapic_v1.method.DEFAULT,
timeout: Union[float, object] = gapic_v1.method.DEFAULT,
metadata: Sequence[Tuple[str, str]] = ()):
"""Instantiate the pager.

Expand All @@ -51,12 +59,17 @@ class {{ method.name }}Pager:
The initial request object.
response (:class:`{{ method.output.ident.sphinx }}`):
The initial response object.
retry (google.api_core.retry.Retry): Designation of what errors,
if any, should be retried.
timeout (float): The timeout for this request.
metadata (Sequence[Tuple[str, str]]): Strings which should be
sent along with the request as metadata.
"""
self._method = method
self._request = {{ method.input.ident }}(request)
self._response = response
self._retry = retry
self._timeout = timeout
self._metadata = metadata

def __getattr__(self, name: str) -> Any:
Expand All @@ -67,7 +80,7 @@ class {{ method.name }}Pager:
yield self._response
while self._response.next_page_token:
self._request.page_token = self._response.next_page_token
self._response = self._method(self._request, metadata=self._metadata)
self._response = self._method(self._request, retry=self._retry, timeout=self._timeout, metadata=self._metadata)
yield self._response

{% if method.paged_result_field.map %}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ from google.api_core import client_options
from google.api_core import exceptions as core_exceptions
from google.api_core import grpc_helpers
from google.api_core import path_template
from google.api_core import retry as retries
{% if service.has_lro %}
from google.api_core import future
from google.api_core import operation
Expand Down Expand Up @@ -938,6 +939,8 @@ def test_{{ method_name }}_pager(transport_name: str = "grpc"):
)

expected_metadata = ()
retry = retries.Retry()
timeout = 5
{% if method.field_headers %}
expected_metadata = tuple(expected_metadata) + (
gapic_v1.routing_header.to_grpc_metadata((
Expand All @@ -949,7 +952,7 @@ def test_{{ method_name }}_pager(transport_name: str = "grpc"):
)),
)
{% endif %}
pager = client.{{ method_name }}(request={})
pager = client.{{ method_name }}(request={}, retry=retry, timeout=timeout)

{% if service.version %}
if HAS_GOOGLE_API_CORE_VERSION_HEADER:
Expand All @@ -958,6 +961,8 @@ def test_{{ method_name }}_pager(transport_name: str = "grpc"):
)
{% endif %}
assert pager._metadata == expected_metadata
assert pager._retry == retry
assert pager._timeout == timeout

results = list(pager)
assert len(results) == 6
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,8 @@
method=rpc,
request=request,
response=response,
retry=retry,
timeout=timeout,
metadata=metadata,
)
{% elif method.extended_lro and full_extended_lro %}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -414,6 +414,8 @@ class {{ service.async_client_name }}:
method=rpc,
request=request,
response=response,
retry=retry,
timeout=timeout,
metadata=metadata,
)
{% endif %}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,15 @@
{# This lives within the loop in order to ensure that this template
is empty if there are no paged methods.
#}
from typing import Any, AsyncIterator, Awaitable, Callable, Sequence, Tuple, Optional, Iterator
from google.api_core import gapic_v1
from google.api_core import retry as retries
from typing import Any, AsyncIterator, Awaitable, Callable, Sequence, Tuple, Optional, Iterator, Union
try:
OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault, None]
OptionalAsyncRetry = Union[retries.AsyncRetry, gapic_v1.method._MethodDefault, None]
except AttributeError: # pragma: NO COVER
OptionalRetry = Union[retries.Retry, object, None] # type: ignore
OptionalAsyncRetry = Union[retries.AsyncRetry, object, None] # type: ignore

{% filter sort_lines %}
{% for method in service.methods.values() | selectattr('paged_result_field') %}
Expand Down Expand Up @@ -43,6 +51,8 @@ class {{ method.name }}Pager:
request: {{ method.input.ident }},
response: {{ method.output.ident }},
*,
retry: OptionalRetry = gapic_v1.method.DEFAULT,
timeout: Union[float, object] = gapic_v1.method.DEFAULT,
metadata: Sequence[Tuple[str, str]] = ()):
"""Instantiate the pager.

Expand All @@ -53,12 +63,17 @@ class {{ method.name }}Pager:
The initial request object.
response ({{ method.output.ident.sphinx }}):
The initial response object.
retry (google.api_core.retry.Retry): Designation of what errors,
if any, should be retried.
timeout (float): The timeout for this request.
metadata (Sequence[Tuple[str, str]]): Strings which should be
sent along with the request as metadata.
"""
self._method = method
self._request = {{ method.input.ident }}(request)
self._response = response
self._retry = retry
self._timeout = timeout
self._metadata = metadata

def __getattr__(self, name: str) -> Any:
Expand All @@ -69,7 +84,7 @@ class {{ method.name }}Pager:
yield self._response
while self._response.next_page_token:
self._request.page_token = self._response.next_page_token
self._response = self._method(self._request, metadata=self._metadata)
self._response = self._method(self._request, retry=self._retry, timeout=self._timeout, metadata=self._metadata)
yield self._response

{% if method.paged_result_field.map %}
Expand Down Expand Up @@ -113,6 +128,8 @@ class {{ method.name }}AsyncPager:
request: {{ method.input.ident }},
response: {{ method.output.ident }},
*,
retry: OptionalAsyncRetry = gapic_v1.method.DEFAULT,
timeout: Union[float, object] = gapic_v1.method.DEFAULT,
metadata: Sequence[Tuple[str, str]] = ()):
"""Instantiates the pager.

Expand All @@ -123,12 +140,17 @@ class {{ method.name }}AsyncPager:
The initial request object.
response ({{ method.output.ident.sphinx }}):
The initial response object.
retry (google.api_core.retry.AsyncRetry): Designation of what errors,
if any, should be retried.
timeout (float): The timeout for this request.
metadata (Sequence[Tuple[str, str]]): Strings which should be
sent along with the request as metadata.
"""
self._method = method
self._request = {{ method.input.ident }}(request)
self._response = response
self._retry = retry
self._timeout = timeout
self._metadata = metadata

def __getattr__(self, name: str) -> Any:
Expand All @@ -139,7 +161,7 @@ class {{ method.name }}AsyncPager:
yield self._response
while self._response.next_page_token:
self._request.page_token = self._response.next_page_token
self._response = await self._method(self._request, metadata=self._metadata)
self._response = await self._method(self._request, retry=self._retry, timeout=self._timeout, metadata=self._metadata)
yield self._response
{% if method.paged_result_field.map %}
def __aiter__(self) -> Iterator[Tuple[str, {{ method.paged_result_field.type.fields.get('value').ident }}]]:
Expand Down
4 changes: 3 additions & 1 deletion gapic/templates/noxfile.py.j2
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,9 @@ def cover(session):
def mypy(session):
"""Run the type checker."""
session.install(
'mypy',
# TODO(https://github.com/googleapis/gapic-generator-python/issues/2066):
# Ignore release of mypy 1.11.0 which may have a regression
'mypy!=1.11.0',
'types-requests',
'types-protobuf'
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ from google.api_core import exceptions as core_exceptions
from google.api_core import grpc_helpers
from google.api_core import grpc_helpers_async
from google.api_core import path_template
from google.api_core import retry as retries
{% if service.has_lro or service.has_extended_lro %}
from google.api_core import future
{% endif %}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -850,6 +850,8 @@ def test_{{ method_name }}_pager(transport_name: str = "grpc"):
)

expected_metadata = ()
retry = retries.Retry()
timeout = 5
{% if not method.explicit_routing and method.field_headers %}
expected_metadata = tuple(expected_metadata) + (
gapic_v1.routing_header.to_grpc_metadata((
Expand All @@ -867,9 +869,11 @@ def test_{{ method_name }}_pager(transport_name: str = "grpc"):
version_header.to_api_version_header("{{ service.version }}"),
)
{% endif %}
pager = client.{{ method_name }}(request={})
pager = client.{{ method_name }}(request={}, retry=retry, timeout=timeout)

assert pager._metadata == expected_metadata
assert pager._retry == retry
assert pager._timeout == timeout

results = list(pager)
assert len(results) == 6
Expand Down
4 changes: 3 additions & 1 deletion noxfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -434,7 +434,9 @@ def showcase_mypy(
"""Perform typecheck analysis on the generated Showcase library."""

# Install pytest and gapic-generator-python
session.install("mypy", "types-pkg-resources", "types-protobuf", "types-requests", "types-dataclasses")
# TODO(https://github.com/googleapis/gapic-generator-python/issues/2066):
# Ignore release of mypy 1.11.0 which may have a regression
session.install("mypy!=1.11.0", "types-pkg-resources", "types-protobuf", "types-requests", "types-dataclasses")

with showcase_library(session, templates=templates, other_opts=other_opts) as lib:
session.chdir(lib)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -478,6 +478,8 @@ async def sample_list_assets():
method=rpc,
request=request,
response=response,
retry=retry,
timeout=timeout,
metadata=metadata,
)

Expand Down Expand Up @@ -1335,6 +1337,8 @@ async def sample_search_all_resources():
method=rpc,
request=request,
response=response,
retry=retry,
timeout=timeout,
metadata=metadata,
)

Expand Down Expand Up @@ -1524,6 +1528,8 @@ async def sample_search_all_iam_policies():
method=rpc,
request=request,
response=response,
retry=retry,
timeout=timeout,
metadata=metadata,
)

Expand Down Expand Up @@ -2276,6 +2282,8 @@ async def sample_list_saved_queries():
method=rpc,
request=request,
response=response,
retry=retry,
timeout=timeout,
metadata=metadata,
)

Expand Down Expand Up @@ -2721,6 +2729,8 @@ async def sample_analyze_org_policies():
method=rpc,
request=request,
response=response,
retry=retry,
timeout=timeout,
metadata=metadata,
)

Expand Down Expand Up @@ -2871,6 +2881,8 @@ async def sample_analyze_org_policy_governed_containers():
method=rpc,
request=request,
response=response,
retry=retry,
timeout=timeout,
metadata=metadata,
)

Expand Down Expand Up @@ -3050,6 +3062,8 @@ async def sample_analyze_org_policy_governed_assets():
method=rpc,
request=request,
response=response,
retry=retry,
timeout=timeout,
metadata=metadata,
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -874,6 +874,8 @@ def sample_list_assets():
method=rpc,
request=request,
response=response,
retry=retry,
timeout=timeout,
metadata=metadata,
)

Expand Down Expand Up @@ -1725,6 +1727,8 @@ def sample_search_all_resources():
method=rpc,
request=request,
response=response,
retry=retry,
timeout=timeout,
metadata=metadata,
)

Expand Down Expand Up @@ -1913,6 +1917,8 @@ def sample_search_all_iam_policies():
method=rpc,
request=request,
response=response,
retry=retry,
timeout=timeout,
metadata=metadata,
)

Expand Down Expand Up @@ -2662,6 +2668,8 @@ def sample_list_saved_queries():
method=rpc,
request=request,
response=response,
retry=retry,
timeout=timeout,
metadata=metadata,
)

Expand Down Expand Up @@ -3104,6 +3112,8 @@ def sample_analyze_org_policies():
method=rpc,
request=request,
response=response,
retry=retry,
timeout=timeout,
metadata=metadata,
)

Expand Down Expand Up @@ -3253,6 +3263,8 @@ def sample_analyze_org_policy_governed_containers():
method=rpc,
request=request,
response=response,
retry=retry,
timeout=timeout,
metadata=metadata,
)

Expand Down Expand Up @@ -3431,6 +3443,8 @@ def sample_analyze_org_policy_governed_assets():
method=rpc,
request=request,
response=response,
retry=retry,
timeout=timeout,
metadata=metadata,
)

Expand Down
Loading

0 comments on commit 76aa98e

Please sign in to comment.