Skip to content

Commit

Permalink
Merge pull request #1090 from markotoplak/proxies-mounts
Browse files Browse the repository at this point in the history
[FIX] import_documents: rewrite proxies to mounts for httpx 0.28
  • Loading branch information
markotoplak authored Dec 6, 2024
2 parents 8b717fe + a2f12fb commit 4293a2c
Showing 1 changed file with 8 additions and 1 deletion.
9 changes: 8 additions & 1 deletion orangecontrib/text/import_documents.py
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,12 @@ def quote_byte(b):
ResponseType = Tuple[Optional[Reader], Optional[TextData], Optional[str]]


def _rewrite_proxies_to_mounts(proxies):
if proxies is None:
return None
return {c: httpx.AsyncHTTPTransport(proxy=url) for c, url in get_proxies().items()}


class UrlProxyReader:
"""
A collection of functions to handle async downloading of a list of documents
Expand Down Expand Up @@ -208,7 +214,8 @@ def read_files(

@staticmethod
async def _read_files(urls: List[str], callback: Callable) -> List[ResponseType]:
async with httpx.AsyncClient(timeout=10.0, proxies=get_proxies()) as client:
proxy_mounts = _rewrite_proxies_to_mounts(get_proxies())
async with httpx.AsyncClient(timeout=10.0, mounts=proxy_mounts) as client:
req = [UrlProxyReader._read_file(url, client, callback) for url in urls]
return await asyncio.gather(*req)

Expand Down

0 comments on commit 4293a2c

Please sign in to comment.