ssl.SSLError: [SSL: APPLICATION_DATA_AFTER_CLOSE_NOTIFY]
application data after close notify
#10358
Open
1 task done
Labels
Describe the bug
While making the large number of concurrent async requests say 100K from service A to another service B, we are getting below error after app. 60000 requests.
Service B is running as 12 replicas behind a AWS load balancer.
Please suggest what is missing. Sample code -
async def process_batch(messages_list,max_tokens,temperature,top_p):
for attempt in range(max_attempts):
try:
connector = aiohttp.TCPConnector(limit = connector_limit) # connector_limit - 2500
timeout = aiohttp.ClientTimeout(
total=max_timeout, # max_timeout - 9000
)
async with aiohttp.ClientSession(connector=connector,timeout=timeout) as session:
batches = [async_chat(session=session,messages=messages) for messages in messages_list]
return await asyncio.gather(*batches)
except Exception as e:
if attempt==max_attempts-1: # max_attempts - 10
raise
await asyncio.sleep(sleep_time) # sleep_time 1 second
aiohttp.client_exceptions.ClientOSError: [Errno 1] [SSL: APPLICATION_DATA_AFTER_CLOSE_NOTIFY] application data after close notify (_ssl.c:2702)
[Errno 1] [SSL: APPLICATION_DATA_AFTER_CLOSE_NOTIFY] application data after close notify (_ssl.c:2702)
2025-01-24 14:08 - aLM_package.scripts.core.utils - ThreadPoolExecutor-72_12 - MainProcess - INFO - [utils.py:121] Error in aLM function -> Traceback (most recent call last):
File "/app/jslenv/lib/python3.10/site-packages/aiohttp/connector.py", line 1115, in _wrap_create_connection
sock = await aiohappyeyeballs.start_connection(
File "/app/jslenv/lib/python3.10/site-packages/aiohappyeyeballs/impl.py", line 104, in start_connection
raise first_exception
File "/app/jslenv/lib/python3.10/site-packages/aiohappyeyeballs/impl.py", line 82, in start_connection
sock = await _connect_sock(
File "/app/jslenv/lib/python3.10/site-packages/aiohappyeyeballs/impl.py", line 174, in _connect_sock
await loop.sock_connect(sock, address)
File "/usr/local/lib/python3.10/asyncio/selector_events.py", line 501, in sock_connect
return await fut
File "/usr/local/lib/python3.10/asyncio/selector_events.py", line 509, in _sock_connect
sock.connect(address)
OSError: [Errno 99] Cannot assign requested address
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/app/src/aLM_package/scripts/core/utils.py", line 113, in async_chat
result = await asyncio.wait_for(make_request(), timeout=max_timeout)
File "/usr/local/lib/python3.10/asyncio/tasks.py", line 445, in wait_for
return fut.result()
File "/app/src/aLM_package/scripts/core/utils.py", line 94, in make_request
async with session.post(url=deployment_url, headers=headers, json=data) as resp:
File "/app/jslenv/lib/python3.10/site-packages/aiohttp/client.py", line 1425, in aenter
self._resp: _RetType = await self._coro
File "/app/jslenv/lib/python3.10/site-packages/aiohttp/client.py", line 703, in _request
conn = await self._connector.connect(
File "/app/jslenv/lib/python3.10/site-packages/aiohttp/connector.py", line 548, in connect
proto = await self._create_connection(req, traces, timeout)
File "/app/jslenv/lib/python3.10/site-packages/aiohttp/connector.py", line 1056, in _create_connection
_, proto = await self._create_direct_connection(req, traces, timeout)
File "/app/jslenv/lib/python3.10/site-packages/aiohttp/connector.py", line 1400, in _create_direct_connection
raise last_exc
File "/app/jslenv/lib/python3.10/site-packages/aiohttp/connector.py", line 1369, in _create_direct_connection
transp, proto = await self._wrap_create_connection(
File "/app/jslenv/lib/python3.10/site-packages/aiohttp/connector.py", line 1130, in _wrap_create_connection
raise client_error(req.connection_key, exc) from exc
aiohttp.client_exceptions.ClientConnectorError: Cannot connect to host process-v1-pod-aLM.domain.ai:443 ssl:default [Cannot assign requested address]
Cannot connect to host process-v1-pod-aLM.domain.ai:443 ssl:default [Cannot assign requested address]
2025-01-24 14:08 - aLM_package.scripts.core.utils - ThreadPoolExecutor-42_11 - MainProcess - INFO - [utils.py:121] Error in aLM function -> Traceback (most recent call last):
File "/app/jslenv/lib/python3.10/site-packages/aiohttp/connector.py", line 1115, in _wrap_create_connection
sock = await aiohappyeyeballs.start_connection(
File "/app/jslenv/lib/python3.10/site-packages/aiohappyeyeballs/impl.py", line 104, in start_connection
raise first_exception
File "/app/jslenv/lib/python3.10/site-packages/aiohappyeyeballs/impl.py", line 82, in start_connection
sock = await _connect_sock(
File "/app/jslenv/lib/python3.10/site-packages/aiohappyeyeballs/impl.py", line 174, in _connect_sock
await loop.sock_connect(sock, address)
File "/usr/local/lib/python3.10/asyncio/selector_events.py", line 501, in sock_connect
return await fut
File "/usr/local/lib/python3.10/asyncio/selector_events.py", line 509, in _sock_connect
sock.connect(address)
OSError: [Errno 99] Cannot assign requested address
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/app/src/aLM_package/scripts/core/utils.py", line 113, in async_chat
result = await asyncio.wait_for(make_request(), timeout=max_timeout)
File "/usr/local/lib/python3.10/asyncio/tasks.py", line 445, in wait_for
return fut.result()
File "/app/src/aLM_package/scripts/core/utils.py", line 94, in make_request
async with session.post(url=deployment_url, headers=headers, json=data) as resp:
File "/app/jslenv/lib/python3.10/site-packages/aiohttp/client.py", line 1425, in aenter
self._resp: _RetType = await self._coro
File "/app/jslenv/lib/python3.10/site-packages/aiohttp/client.py", line 703, in _request
conn = await self._connector.connect(
File "/app/jslenv/lib/python3.10/site-packages/aiohttp/connector.py", line 548, in connect
proto = await self._create_connection(req, traces, timeout)
File "/app/jslenv/lib/python3.10/site-packages/aiohttp/connector.py", line 1056, in _create_connection
_, proto = await self._create_direct_connection(req, traces, timeout)
File "/app/jslenv/lib/python3.10/site-packages/aiohttp/connector.py", line 1400, in _create_direct_connection
raise last_exc
File "/app/jslenv/lib/python3.10/site-packages/aiohttp/connector.py", line 1369, in _create_direct_connection
transp, proto = await self._wrap_create_connection(
File "/app/jslenv/lib/python3.10/site-packages/aiohttp/connector.py", line 1130, in _wrap_create_connection
raise client_error(req.connection_key, exc) from exc
aiohttp.client_exceptions.ClientConnectorError: Cannot connect to host process-v1-pod-aLM.domain.ai:443 ssl:default [Cannot assign requested address]
Cannot connect to host process-v1-pod-aLM.domain.ai:443 ssl:default [Cannot assign requested address]
2025-01-24 14:08 - aLM_package.scripts.core.utils - ThreadPoolExecutor-84_1 - MainProcess - INFO - [utils.py:121] Error in aLM function -> Traceback (most recent call last):
File "/app/src/aLM_package/scripts/core/utils.py", line 113, in async_chat
result = await asyncio.wait_for(make_request(), timeout=max_timeout)
File "/usr/local/lib/python3.10/asyncio/tasks.py", line 445, in wait_for
return fut.result()
File "/app/src/aLM_package/scripts/core/utils.py", line 94, in make_request
async with session.post(url=deployment_url, headers=headers, json=data) as resp:
File "/app/jslenv/lib/python3.10/site-packages/aiohttp/client.py", line 1425, in aenter
self._resp: _RetType = await self._coro
File "/app/jslenv/lib/python3.10/site-packages/aiohttp/client.py", line 703, in _request
conn = await self._connector.connect(
File "/app/jslenv/lib/python3.10/site-packages/aiohttp/connector.py", line 558, in connect
raise ClientConnectionError("Connector is closed.")
aiohttp.client_exceptions.ClientConnectionError: Connector is closed.
Connector is closed.
2025-01-24 14:08 - aLM_package.scripts.core.utils - ThreadPoolExecutor-51_29 - MainProcess - INFO - [utils.py:121] Error in aLM function -> Traceback (most recent call last):
File "/usr/local/lib/python3.10/asyncio/sslproto.py", line 534, in data_received
ssldata, appdata = self._sslpipe.feed_ssldata(data)
File "/usr/local/lib/python3.10/asyncio/sslproto.py", line 206, in feed_ssldata
self._sslobj.unwrap()
File "/usr/local/lib/python3.10/ssl.py", line 979, in unwrap
return self._sslobj.shutdown()
ssl.SSLError: [SSL: APPLICATION_DATA_AFTER_CLOSE_NOTIFY] application data after close notify (_ssl.c:2702)
To Reproduce
async def process_batch(messages_list,max_tokens,temperature,top_p):
for attempt in range(max_attempts):
try:
connector = aiohttp.TCPConnector(limit = connector_limit) # connector_limit - 2500
timeout = aiohttp.ClientTimeout(
total=max_timeout, # max_timeout - 9000
)
async with aiohttp.ClientSession(connector=connector,timeout=timeout) as session:
batches = [async_chat(session=session,messages=messages) for messages in messages_list]
return await asyncio.gather(*batches)
except Exception as e:
if attempt==max_attempts-1: # max_attempts - 10
raise
await asyncio.sleep(sleep_time) # sleep_time 1 second
Expected behavior
asyn calls should scale to millions
Logs/tracebacks
Python Version
aiohttp Version
multidict Version
propcache Version
yarl Version
OS
bash-5.2# which OS
which: no OS in (/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin)
Related component
Client
Additional context
ash-5.2# pip3 freeze
aiobotocore==2.5.0
aiohappyeyeballs==2.4.0
aiohttp==3.10.2
aioitertools==0.11.0
aiosignal==1.3.1
aniso8601==9.0.1
annotated-types==0.7.0
apipkg==3.0.1
asgiref==3.8.1
astor==0.8.1
astroid==2.15.5
async-timeout==4.0.2
atomicwrites==1.4.1
attrs==23.1.0
autograd==1.5
autograd-gamma==0.5.0
autopep8==2.0.2
backcall==0.2.0
blinker==1.6.2
blis==0.7.11
boto3==1.26.76
botocore==1.29.76
cachetools==5.3.0
catalogue==2.0.10
certifi==2024.7.4
chardet==5.1.0
charset-normalizer==3.1.0
click==8.1.3
cloudpathlib==0.14.0
cloudpickle==2.2.1
colorama==0.4.6
commonmark==0.9.1
contourpy==1.0.7
cycler==0.11.0
cymem==2.0.11
decorator==5.1.1
deepdiff==6.3.0
dill==0.3.6
Django==5.1.1
django-phonenumber-field==7.1.0
django-phonenumbers==1.0.1
django-webpack-loader==1.8.1
djangorestframework==3.15.2
djangorestframework-jwt==1.0.1
dnspython==2.6.1
docutils==0.20
dpath==2.1.5
et-xmlfile==1.1.0
eventlet==0.35.2
exceptiongroup==1.1.1
execnet==1.9.0
Faker==18.7.0
Flask==2.3.2
flask-restplus==0.13.0
fonttools==4.43.0
formulaic==0.6.1
frozenlist==1.3.3
fsspec==2023.5.0
funcy==2.0
future==0.18.3
google-api-core==2.11.0
google-auth==2.18.0
google-cloud-bigquery==3.10.0
google-cloud-core==2.3.2
google-crc32c==1.5.0
google-resumable-media==2.5.0
googleapis-common-protos==1.59.0
GPUtil==1.4.0
greenlet==2.0.2
grpcio==1.56.2
grpcio-status==1.54.2
gunicorn==22.0.0
idna==3.7
importlib-metadata==6.6.0
iniconfig==2.0.0
interface-meta==1.3.0
ipython-genutils==0.2.0
isort==5.12.0
itsdangerous==2.1.2
jedi==0.18.2
Jinja2==3.1.4
jmespath==1.0.1
joblib==1.2.0
jsonpickle==3.0.1
jsonschema==4.17.3
kiwisolver==1.4.4
lazy-object-proxy==1.9.0
libmagic==1.0
lifelines==0.27.7
lxml==4.9.2
markdown-it-py==2.2.0
MarkupSafe==2.1.2
matplotlib==3.7.1
mccabe==0.7.0
mdurl==0.1.2
mod_wsgi==4.9.4
monotonic==1.6
more-itertools==9.1.0
multidict==6.0.4
murmurhash==1.0.12
numpy==1.24.3
numpyencoder==0.3.0
openpyxl==3.1.2
ordered-set==4.1.0
orjson==3.9.15
packaging==23.1
pandas==2.0.1
parso==0.8.3
pathlib_abc==0.1.1
pathy==0.11.0
pep8==1.7.1
pexpect==4.8.0
phonenumbers==8.13.11
pickleshare==0.7.5
pillow==10.3.0
platformdirs==3.5.1
pluggy==1.0.0
polling2==0.5.0
preshed==3.0.9
prompt-toolkit==3.0.38
proto-plus==1.22.2
protobuf==4.23.0
psycopg2-binary==2.9.6
ptyprocess==0.7.0
py==1.11.0
py4j==0.10.9.7
pyasn1==0.5.0
pyasn1-modules==0.3.0
pycodestyle==2.10.0
pycryptodome==3.19.1
pydantic==1.8.2
pydantic_core==2.10.0
Pygments==2.15.1
PyJWT==2.4.0
pylint==2.17.4
pylint-django==2.5.3
pylint-flask==0.6
pylint-plugin-utils==0.8.1
pymssql==2.2.7
PyMuPDF==1.22.3
pyparsing==3.0.9
pyrsistent==0.19.3
pytest==7.3.1
pytest-flask==1.2.0
pytest-forked==1.6.0
pytest-xdist==3.3.0
python-dateutil==2.8.2
python-magic==0.4.27
pytoolconfig==1.2.5
pytz==2023.3
PyYAML==6.0
rapidfuzz==3.0.0
requests==2.32.0
requests-aws4auth==1.2.3
rich==13.3.5
rope==1.8.0
rsa==4.9
rwslib==1.2.9
s3fs==2023.5.0
s3transfer==0.6.1
scikit-learn==1.5.0
scipy==1.10.1
six==1.16.0
smart-open==6.3.0
spacy==3.1.4
spacy-legacy==3.0.12
SQLAlchemy==2.0.13
sqlparse==0.5.0
sqlvalidator==0.0.20
srsly==2.5.1
text-unidecode==1.3
thinc==8.0.17
threadpoolctl==3.1.0
tomli==2.0.1
tomlkit==0.11.8
tqdm==4.67.1
traitlets==5.9.0
typed-ast==1.5.4
typer==0.4.2
typing_extensions==4.12.2
tzdata==2023.3
urllib3==1.26.19
wasabi==0.10.1
wcwidth==0.2.6
Werkzeug==3.0.3
wrapt==1.15.0
xlrd==2.0.1
xlwt==1.3.0
xmltodict3==0.0.4
yarl==1.9.2
zipp==3.19.1
Code of Conduct
The text was updated successfully, but these errors were encountered: