From c898cfb62b86fba80d45fb013f078236a0263a5b Mon Sep 17 00:00:00 2001 From: ruanweihong-aaa <310496494+ruanweihong-aaa@users.noreply.github.com> Date: Mon, 10 Aug 2026 23:56:47 +0800 Subject: [PATCH 1/2] feat(ai-proxy): support phase-specific HTTP timeouts Add optional connect, send, and read timeout settings for ai-proxy and ai-proxy-multi while preserving the existing timeout fallback behavior. --- apisix/plugins/ai-proxy/base.lua | 11 +- apisix/plugins/ai-proxy/schema.lua | 30 ++++ apisix/plugins/ai-transport/http.lua | 10 +- docs/en/latest/plugins/ai-proxy-multi.md | 5 +- docs/en/latest/plugins/ai-proxy.md | 5 +- docs/zh/latest/plugins/ai-proxy-multi.md | 5 +- docs/zh/latest/plugins/ai-proxy.md | 5 +- t/plugin/ai-proxy-multi.t | 67 ++++++++ t/plugin/ai-proxy-timeout-callsite.t | 136 +++++++++++++++++ t/plugin/ai-proxy.t | 186 +++++++++++++++++++++++ t/plugin/ai-transport-http.t | 90 +++++++++++ 11 files changed, 543 insertions(+), 7 deletions(-) create mode 100644 t/plugin/ai-proxy-timeout-callsite.t diff --git a/apisix/plugins/ai-proxy/base.lua b/apisix/plugins/ai-proxy/base.lua index fa815f430fb5..bea1b833e0a1 100644 --- a/apisix/plugins/ai-proxy/base.lua +++ b/apisix/plugins/ai-proxy/base.lua @@ -341,7 +341,16 @@ function _M.before_proxy(conf, ctx, on_error) core.json.delay_encode(log_sanitize.redact_params(params), true)) -- Step 4: Send via transport - local res, transport_err, err_meta = transport_http.request(params, conf.timeout) + local timeout = conf.timeout + if conf.connect_timeout or conf.send_timeout or conf.read_timeout then + timeout = { + connect_timeout = conf.connect_timeout or conf.timeout, + send_timeout = conf.send_timeout or conf.timeout, + read_timeout = conf.read_timeout or conf.timeout, + } + end + + local res, transport_err, err_meta = transport_http.request(params, timeout) if not res then core.log.warn("failed to send request to LLM server: ", transport_err) if err_meta then diff --git a/apisix/plugins/ai-proxy/schema.lua b/apisix/plugins/ai-proxy/schema.lua index d5f13751d000..42a10bb2ec41 100644 --- a/apisix/plugins/ai-proxy/schema.lua +++ b/apisix/plugins/ai-proxy/schema.lua @@ -344,6 +344,21 @@ _M.ai_proxy_schema = { default = 30000, description = "timeout in milliseconds", }, + connect_timeout = { + type = "integer", + minimum = 1, + maximum = 600000, + }, + send_timeout = { + type = "integer", + minimum = 1, + maximum = 600000, + }, + read_timeout = { + type = "integer", + minimum = 1, + maximum = 600000, + }, max_req_body_size = { type = "integer", minimum = 1, @@ -478,6 +493,21 @@ _M.ai_proxy_multi_schema = { default = 30000, description = "timeout in milliseconds", }, + connect_timeout = { + type = "integer", + minimum = 1, + maximum = 600000, + }, + send_timeout = { + type = "integer", + minimum = 1, + maximum = 600000, + }, + read_timeout = { + type = "integer", + minimum = 1, + maximum = 600000, + }, max_req_body_size = { type = "integer", minimum = 1, diff --git a/apisix/plugins/ai-transport/http.lua b/apisix/plugins/ai-transport/http.lua index a2075aa3f4c3..a2fa99485549 100644 --- a/apisix/plugins/ai-transport/http.lua +++ b/apisix/plugins/ai-transport/http.lua @@ -94,7 +94,8 @@ end -- @param params table HTTP request parameters: -- {method, scheme, host, port, path, headers, query, body (table), -- ssl_verify, ssl_server_name} --- @param timeout number Request timeout in milliseconds +-- @param timeout number|table Request timeout in milliseconds, or a table with +-- {connect_timeout, send_timeout, read_timeout} in milliseconds -- @return table|nil Response object (with body_reader, headers, status, -- _upstream_addr, _upstream_uri, _connect_time, _header_time, _t0) -- @return string|nil Error message @@ -104,7 +105,12 @@ function _M.request(params, timeout) if not httpc then return nil, "failed to create http client: " .. (err or "unknown") end - httpc:set_timeout(timeout) + if type(timeout) == "table" then + httpc:set_timeouts(timeout.connect_timeout, timeout.send_timeout, + timeout.read_timeout) + else + httpc:set_timeout(timeout) + end local upstream_addr = (params.host or "") .. ":" .. (params.port or "") local upstream_host = params.host or "" diff --git a/docs/en/latest/plugins/ai-proxy-multi.md b/docs/en/latest/plugins/ai-proxy-multi.md index dc58ca565fbf..7aa33609ac4e 100644 --- a/docs/en/latest/plugins/ai-proxy-multi.md +++ b/docs/en/latest/plugins/ai-proxy-multi.md @@ -137,7 +137,10 @@ When an instance's `provider` is set to `bedrock`, the Plugin expects requests i | semantic_opts.embeddings.auth.query | object | False | | | Authentication query parameters. Encrypted at rest. | | semantic_opts.embeddings.timeout | integer | False | 3000 | minimum = 1 | Embedding request timeout in milliseconds. The query prompt is embedded synchronously on every request, so this bounds the latency added to each request when the embedding endpoint is slow or down (the request then fails open to the fallback). | | semantic_opts.embeddings.ssl_verify | boolean | False | true | | If true, verify the embedding service's certificate. | -| timeout | integer | False | 30000 | greater than or equal to 1 | Request timeout in milliseconds when requesting the LLM service. Applied per socket operation (connect / send / read block); does not cap the total duration of a streaming response. | +| timeout | integer | False | 30000 | 1 - 600000 | Default request timeout in milliseconds when requesting the LLM service. It remains the fallback for each phase below and is used for all phases when no phase-specific timeout is configured. | +| connect_timeout | integer | False | | 1 - 600000 | Connect timeout in milliseconds. Falls back to `timeout` when unset. | +| send_timeout | integer | False | | 1 - 600000 | Send timeout in milliseconds. Falls back to `timeout` when unset. | +| read_timeout | integer | False | | 1 - 600000 | Read timeout in milliseconds. Falls back to `timeout` when unset. This applies per socket read and does not cap the total duration of a streaming response. | | max_req_body_size | integer | False | 67108864 | greater than or equal to 1 | Maximum request body size in bytes that the plugin reads into memory. Requests whose body exceeds this limit are rejected with `413`. Prevents unbounded memory buffering of large request bodies. | | max_stream_duration_ms | integer | False | | greater than or equal to 1 | Maximum wall-clock duration (in milliseconds) for a streaming AI response. If the upstream keeps sending data past this deadline, the gateway closes the connection. Unset means no cap. Use this to protect the gateway from upstream bugs that produce tokens indefinitely. When the limit is hit mid-stream, the downstream SSE stream is truncated (no protocol-specific terminator such as `[DONE]`, `message_stop`, or `response.completed`); well-behaved clients should treat a missing terminator as an incomplete response. | | max_response_bytes | integer | False | | greater than or equal to 1 | Maximum total bytes read from the upstream for a single AI response (streaming or non-streaming). If exceeded, the gateway closes the connection. For non-streaming responses with `Content-Length`, the check is performed before reading the body; for chunked (no-`Content-Length`) non-streaming responses and for streaming responses, the cap is enforced incrementally as bytes are received. Unset means no cap. | diff --git a/docs/en/latest/plugins/ai-proxy.md b/docs/en/latest/plugins/ai-proxy.md index b93702ed00af..85180b8bad8f 100644 --- a/docs/en/latest/plugins/ai-proxy.md +++ b/docs/en/latest/plugins/ai-proxy.md @@ -93,7 +93,10 @@ When `provider` is set to `bedrock`, the Plugin expects requests in the [Bedrock | logging | object | False | | | Logging configurations. Does not affect `error.log`. | | logging.summaries | boolean | False | false | | If true, logs request LLM model, duration, request, and response tokens. | | logging.payloads | boolean | False | false | | If true, logs request and response payload. | -| timeout | integer | False | 30000 | 1 - 600000 | Request timeout in milliseconds when requesting the LLM service. Applied per socket operation (connect / send / read block); does not cap the total duration of a streaming response. | +| timeout | integer | False | 30000 | 1 - 600000 | Default request timeout in milliseconds when requesting the LLM service. It remains the fallback for each phase below and is used for all phases when no phase-specific timeout is configured. | +| connect_timeout | integer | False | | 1 - 600000 | Connect timeout in milliseconds. Falls back to `timeout` when unset. | +| send_timeout | integer | False | | 1 - 600000 | Send timeout in milliseconds. Falls back to `timeout` when unset. | +| read_timeout | integer | False | | 1 - 600000 | Read timeout in milliseconds. Falls back to `timeout` when unset. This applies per socket read and does not cap the total duration of a streaming response. | | max_stream_duration_ms | integer | False | | ≥ 1 | Maximum wall-clock duration (in milliseconds) for a streaming AI response. If the upstream keeps sending data past this deadline, the gateway closes the connection. Unset means no cap. Use this to protect the gateway from upstream bugs that produce tokens indefinitely. When the limit is hit mid-stream, the downstream SSE stream is truncated (no protocol-specific terminator such as `[DONE]`, `message_stop`, or `response.completed`); well-behaved clients should treat a missing terminator as an incomplete response. | | max_response_bytes | integer | False | | ≥ 1 | Maximum total bytes read from the upstream for a single AI response (streaming or non-streaming). If exceeded, the gateway closes the connection. For non-streaming responses with `Content-Length`, the check is performed before reading the body; for chunked (no-`Content-Length`) non-streaming responses and for streaming responses, the cap is enforced incrementally as bytes are received. Unset means no cap. | | max_req_body_size | integer | False | 67108864 | >= 1 | Maximum request body size in bytes that the plugin reads into memory. Requests whose body exceeds this limit are rejected with `413`. Prevents unbounded memory buffering of large request bodies. | diff --git a/docs/zh/latest/plugins/ai-proxy-multi.md b/docs/zh/latest/plugins/ai-proxy-multi.md index 686ad7459eda..6fbe11810a4c 100644 --- a/docs/zh/latest/plugins/ai-proxy-multi.md +++ b/docs/zh/latest/plugins/ai-proxy-multi.md @@ -139,7 +139,10 @@ import TabItem from '@theme/TabItem'; | semantic_opts.embeddings.auth.query | object | 否 | | | 认证查询参数。加密存储。 | | semantic_opts.embeddings.timeout | integer | 否 | 3000 | 最小值为 1 | 嵌入请求的超时时间（毫秒）。每个请求都会同步嵌入查询提示词，因此该值限定了当嵌入端点变慢或不可用时每个请求增加的延迟上限（随后请求会 fail-open 到回退实例）。 | | semantic_opts.embeddings.ssl_verify | boolean | 否 | true | | 如果为 true，验证嵌入服务的证书。 | -| timeout | integer | 否 | 30000 | 大于或等于 1 | 请求 LLM 服务时的请求超时时间（毫秒）。应用于单次 socket 操作（连接 / 发送 / 读取块），不限制流式响应的总时长。 | +| timeout | integer | 否 | 30000 | 1 - 600000 | 请求 LLM 服务时的默认超时时间（毫秒）。它是下列各阶段的回退值；未配置任一阶段专用超时时间时，所有阶段均使用该值。 | +| connect_timeout | integer | 否 | | 1 - 600000 | 连接超时时间（毫秒）。未设置时回退到 `timeout`。 | +| send_timeout | integer | 否 | | 1 - 600000 | 发送超时时间（毫秒）。未设置时回退到 `timeout`。 | +| read_timeout | integer | 否 | | 1 - 600000 | 读取超时时间（毫秒）。未设置时回退到 `timeout`。它针对每次 socket 读取，不限制流式响应的总时长。 | | max_stream_duration_ms | integer | 否 | | 大于或等于 1 | 流式 AI 响应的总墙钟时长上限（毫秒）。若上游在此时间后仍持续发送数据，网关将关闭连接。未设置时不限制。用于防护上游持续输出 token 导致网关 CPU 被打满的异常情况。中途触发上限时，下游 SSE 流会被截断（不再发送协议特定的终止标记，例如 `[DONE]`、`message_stop` 或 `response.completed`），客户端应将缺失的终止标记视为响应未完成。 | | max_response_bytes | integer | 否 | | 大于或等于 1 | 单次 AI 响应（流式或非流式）允许从上游读取的最大总字节数。超出时关闭连接。非流式响应若存在 `Content-Length`，在读取 body 之前预检；否则（chunked 传输）与流式响应一样在接收字节的过程中增量检查。未设置时不限制。 | | keepalive | boolean | 否 | true | | 如果为 true，在请求 LLM 服务时保持连接活跃。 | diff --git a/docs/zh/latest/plugins/ai-proxy.md b/docs/zh/latest/plugins/ai-proxy.md index 62a1b4eae41e..4ddc194782f2 100644 --- a/docs/zh/latest/plugins/ai-proxy.md +++ b/docs/zh/latest/plugins/ai-proxy.md @@ -93,7 +93,10 @@ import TabItem from '@theme/TabItem'; | logging | object | 否 | | | 日志配置。不影响 `error.log`。 | | logging.summaries | boolean | 否 | false | | 如果为 true，记录请求 LLM 模型、持续时间、请求和响应令牌。 | | logging.payloads | boolean | 否 | false | | 如果为 true，记录请求和响应负载。 | -| timeout | integer | 否 | 30000 | 1 - 600000 | 请求 LLM 服务时的请求超时时间（毫秒）。按单次 socket 操作（连接 / 发送 / 读取数据块）计算，不限制流式响应的总时长。 | +| timeout | integer | 否 | 30000 | 1 - 600000 | 请求 LLM 服务时的默认超时时间（毫秒）。它是下列各阶段的回退值；未配置任一阶段专用超时时间时，所有阶段均使用该值。 | +| connect_timeout | integer | 否 | | 1 - 600000 | 连接超时时间（毫秒）。未设置时回退到 `timeout`。 | +| send_timeout | integer | 否 | | 1 - 600000 | 发送超时时间（毫秒）。未设置时回退到 `timeout`。 | +| read_timeout | integer | 否 | | 1 - 600000 | 读取超时时间（毫秒）。未设置时回退到 `timeout`。它针对每次 socket 读取，不限制流式响应的总时长。 | | max_stream_duration_ms | integer | 否 | | ≥ 1 | 流式 AI 响应的最大墙钟时长（毫秒）。如果上游在该截止时间后仍持续发送数据，网关会关闭连接。不设置表示不限制。用于防止上游异常无限产出 token。当在流式过程中触发该限制时，下游 SSE 流会被截断（不会发送 `[DONE]`、`message_stop`、`response.completed` 等协议终止标记）；行为正常的客户端应将缺少终止标记视为不完整的响应。 | | max_response_bytes | integer | 否 | | ≥ 1 | 单次 AI 响应（流式或非流式）从上游读取的最大总字节数。超过则网关关闭连接。对于带 `Content-Length` 的非流式响应，在读取响应体前进行检查；对于分块（无 `Content-Length`）的非流式响应以及流式响应，则在接收字节的过程中增量地强制执行该上限。不设置表示不限制。 | | keepalive | boolean | 否 | true | | 如果为 true，在请求 LLM 服务时保持连接活跃。 | diff --git a/t/plugin/ai-proxy-multi.t b/t/plugin/ai-proxy-multi.t index c2ecdcd4ff8e..85ed17a26462 100644 --- a/t/plugin/ai-proxy-multi.t +++ b/t/plugin/ai-proxy-multi.t @@ -77,6 +77,73 @@ passed +=== TEST 1a: phase timeouts accept valid values and reject invalid values +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.ai-proxy-multi") + + local function base_conf() + return { + instances = { + { + name = "openai-official", + provider = "openai", + options = {model = "gpt-4"}, + weight = 1, + auth = {header = {some_header = "some_value"}}, + } + } + } + end + + local valid_cases = { + {}, + {connect_timeout = 1, send_timeout = 1, read_timeout = 1}, + {connect_timeout = 600000, send_timeout = 600000, read_timeout = 600000}, + {connect_timeout = 101, send_timeout = 202, read_timeout = 303}, + } + for _, timeouts in ipairs(valid_cases) do + local conf = base_conf() + for key, value in pairs(timeouts) do + conf[key] = value + end + assert(plugin.check_schema(conf)) + ngx.say("valid") + end + + local invalid_values = {0, 600001, 1.5, "invalid"} + for _, key in ipairs({"connect_timeout", "send_timeout", "read_timeout"}) do + for _, value in ipairs(invalid_values) do + local conf = base_conf() + conf[key] = value + local ok = plugin.check_schema(conf) + assert(not ok) + ngx.say("invalid: ", key) + end + end + } + } +--- response_body +valid +valid +valid +valid +invalid: connect_timeout +invalid: connect_timeout +invalid: connect_timeout +invalid: connect_timeout +invalid: send_timeout +invalid: send_timeout +invalid: send_timeout +invalid: send_timeout +invalid: read_timeout +invalid: read_timeout +invalid: read_timeout +invalid: read_timeout + + + === TEST 2: unsupported provider --- config location /t { diff --git a/t/plugin/ai-proxy-timeout-callsite.t b/t/plugin/ai-proxy-timeout-callsite.t new file mode 100644 index 000000000000..f1739cd70a07 --- /dev/null +++ b/t/plugin/ai-proxy-timeout-callsite.t @@ -0,0 +1,136 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +repeat_each(1); +no_long_string(); +no_root_location(); + +run_tests; + +__DATA__ + +=== TEST 1: ai-proxy resolves phase timeout fallbacks at the transport call site +--- config + location /t { + content_by_lua_block { + local provider_name = "apisix.plugins.ai-providers.timeout-capture" + local transport_name = "apisix.plugins.ai-transport.http" + local exporter_name = "apisix.plugins.prometheus.exporter" + local sanitize_name = "apisix.utils.log-sanitize" + local base_name = "apisix.plugins.ai-proxy.base" + + local saved = {} + for _, name in ipairs({ + provider_name, + transport_name, + exporter_name, + sanitize_name, + base_name, + }) do + saved[name] = package.loaded[name] + end + + local observed + package.loaded[provider_name] = { + capabilities = { + ["openai-chat"] = { + path = "/v1/chat/completions", + host = "127.0.0.1", + }, + }, + build_body = function(_, body) + return body, true + end, + build_request = function(_, _, body) + return { + method = "POST", + host = "127.0.0.1", + port = 80, + path = "/v1/chat/completions", + headers = {}, + body = body, + } + end, + } + package.loaded[transport_name] = { + request = function(_, timeout) + observed = timeout + return nil, "captured timeout" + end, + handle_error = function() + return 504 + end, + } + package.loaded[exporter_name] = { + inc_llm_active_connections = function() end, + } + package.loaded[sanitize_name] = { + redact_params = function(params) + return params + end, + } + package.loaded[base_name] = nil + + local base = require(base_name) + local cases = { + {name = "omitted", conf = {timeout = 900}}, + {name = "connect", conf = {timeout = 900, connect_timeout = 101}}, + {name = "send", conf = {timeout = 900, send_timeout = 202}}, + {name = "read", conf = {timeout = 900, read_timeout = 303}}, + } + + for _, case in ipairs(cases) do + observed = nil + local ctx = { + ai_client_protocol = "openai-chat", + picked_ai_instance = { + name = "capture", + provider = "timeout-capture", + options = {model = "gpt-4"}, + }, + var = {uri = "/t"}, + } + + ngx.ctx.api_ctx = ctx + base.before_proxy(case.conf, ctx) + if type(observed) == "number" then + ngx.say(case.name, ":number:", observed) + else + ngx.say(case.name, ":table:", + observed.connect_timeout, ",", + observed.send_timeout, ",", + observed.read_timeout) + end + end + ngx.ctx.api_ctx = nil + + for name, value in pairs(saved) do + package.loaded[name] = value + end + } + } +--- request +POST /t +{"model":"gpt-4","messages":[{"role":"user","content":"hello"}]} +--- more_headers +Content-Type: application/json +--- response_body +omitted:number:900 +connect:table:101,900,900 +send:table:900,202,900 +read:table:900,900,303 diff --git a/t/plugin/ai-proxy.t b/t/plugin/ai-proxy.t index 8f57acae6d9f..4a44795c9db7 100644 --- a/t/plugin/ai-proxy.t +++ b/t/plugin/ai-proxy.t @@ -69,6 +69,67 @@ passed +=== TEST 1a: phase timeouts accept valid values and reject invalid values +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.ai-proxy") + + local function base_conf() + return { + provider = "openai", + options = {model = "gpt-4"}, + auth = {header = {some_header = "some_value"}}, + } + end + + local valid_cases = { + {}, + {connect_timeout = 1, send_timeout = 1, read_timeout = 1}, + {connect_timeout = 600000, send_timeout = 600000, read_timeout = 600000}, + {connect_timeout = 101, send_timeout = 202, read_timeout = 303}, + } + for _, timeouts in ipairs(valid_cases) do + local conf = base_conf() + for key, value in pairs(timeouts) do + conf[key] = value + end + assert(plugin.check_schema(conf)) + ngx.say("valid") + end + + local invalid_values = {0, 600001, 1.5, "invalid"} + for _, key in ipairs({"connect_timeout", "send_timeout", "read_timeout"}) do + for _, value in ipairs(invalid_values) do + local conf = base_conf() + conf[key] = value + local ok = plugin.check_schema(conf) + assert(not ok) + ngx.say("invalid: ", key) + end + end + } + } +--- response_body +valid +valid +valid +valid +invalid: connect_timeout +invalid: connect_timeout +invalid: connect_timeout +invalid: connect_timeout +invalid: send_timeout +invalid: send_timeout +invalid: send_timeout +invalid: send_timeout +invalid: read_timeout +invalid: read_timeout +invalid: read_timeout +invalid: read_timeout + + + === TEST 2: unsupported provider --- config location /t { @@ -1325,3 +1386,128 @@ POST /anything --- response_body_like: upstream boom --- response_headers Content-Type: application/json + + + +=== TEST 40: configure independent timeouts with a short read phase +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/anything", + "plugins": { + "ai-proxy": { + "provider": "openai-compatible", + "auth": { + "header": { + "Authorization": "Bearer token" + } + }, + "options": { + "model": "delayed" + }, + "override": { + "endpoint": "http://127.0.0.1:6726/v1/chat/completions" + }, + "timeout": 1000, + "read_timeout": 50, + "ssl_verify": false + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 41: short read phase expires while connect and send phases are longer +--- http_config + server { + server_name delayed_ai; + listen 6726; + default_type 'application/json'; + location / { + content_by_lua_block { + ngx.sleep(0.25) + ngx.say([[{"choices":[{"message":{"content":"delayed"}}]}]]) + } + } + } +--- request +POST /anything +{ "messages": [ { "role": "user", "content": "slow response"} ] } +--- error_code: 504 + + + +=== TEST 42: configure a read phase that exceeds the local upstream delay +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/anything", + "plugins": { + "ai-proxy": { + "provider": "openai-compatible", + "auth": { + "header": { + "Authorization": "Bearer token" + } + }, + "options": { + "model": "delayed" + }, + "override": { + "endpoint": "http://127.0.0.1:6726/v1/chat/completions" + }, + "timeout": 1000, + "read_timeout": 1000, + "ssl_verify": false + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 43: longer read phase receives the delayed local upstream response +--- http_config + server { + server_name delayed_ai; + listen 6726; + default_type 'application/json'; + location / { + content_by_lua_block { + ngx.sleep(0.25) + ngx.say([[{"choices":[{"message":{"content":"delayed"}}]}]]) + } + } + } +--- request +POST /anything +{ "messages": [ { "role": "user", "content": "slow response"} ] } +--- error_code: 200 +--- response_body_like: delayed diff --git a/t/plugin/ai-transport-http.t b/t/plugin/ai-transport-http.t index b475164939d3..57c516dc7d05 100644 --- a/t/plugin/ai-transport-http.t +++ b/t/plugin/ai-transport-http.t @@ -353,3 +353,93 @@ connect: operation timed out => 504 connect: Operation timed out => 504 request: connection refused => 500 request: connection reset by peer => 500 + + + +=== TEST 8: AI transport applies independent connect send and read timeouts +--- config + location /t { + content_by_lua_block { + local orig_http = package.loaded["resty.http"] + local orig_transport = package.loaded["apisix.plugins.ai-transport.http"] + + package.loaded["resty.http"] = { + new = function() + return { + set_timeout = function() + error("legacy set_timeout must not be used") + end, + set_timeouts = function(_, connect, send, read) + ngx.say(connect, ",", send, ",", read) + end, + connect = function() return true end, + request = function() return {headers = {}, status = 200} end, + } + end, + } + + package.loaded["apisix.plugins.ai-transport.http"] = nil + local transport = require("apisix.plugins.ai-transport.http") + local res, err = transport.request({ + host = "127.0.0.1", + port = 80, + path = "/", + body = {}, + }, { + connect_timeout = 101, + send_timeout = 202, + read_timeout = 303, + }) + if not res then + ngx.say(err) + end + + package.loaded["resty.http"] = orig_http + package.loaded["apisix.plugins.ai-transport.http"] = orig_transport + } + } +--- response_body +101,202,303 + + + +=== TEST 9: AI transport keeps numeric timeout callers on set_timeout +--- config + location /t { + content_by_lua_block { + local orig_http = package.loaded["resty.http"] + local orig_transport = package.loaded["apisix.plugins.ai-transport.http"] + + package.loaded["resty.http"] = { + new = function() + return { + set_timeout = function(_, timeout) + ngx.say(timeout) + end, + set_timeouts = function() + error("set_timeouts must not be used for numeric callers") + end, + connect = function() return true end, + request = function() return {headers = {}, status = 200} end, + } + end, + } + + package.loaded["apisix.plugins.ai-transport.http"] = nil + local transport = require("apisix.plugins.ai-transport.http") + local res, err = transport.request({ + host = "127.0.0.1", + port = 80, + path = "/", + body = {}, + }, 456) + if not res then + ngx.say(err) + end + + package.loaded["resty.http"] = orig_http + package.loaded["apisix.plugins.ai-transport.http"] = orig_transport + } + } +--- response_body +456 From 84c4343e4b3ccb10db77860e2df28d67d4df723a Mon Sep 17 00:00:00 2001 From: ruanweihong-aaa <310496494+ruanweihong-aaa@users.noreply.github.com> Date: Tue, 11 Aug 2026 12:11:27 +0800 Subject: [PATCH 2/2] test(ai-proxy): remove mock-only timeout test --- t/plugin/ai-proxy-timeout-callsite.t | 136 --------------------------- 1 file changed, 136 deletions(-) delete mode 100644 t/plugin/ai-proxy-timeout-callsite.t diff --git a/t/plugin/ai-proxy-timeout-callsite.t b/t/plugin/ai-proxy-timeout-callsite.t deleted file mode 100644 index f1739cd70a07..000000000000 --- a/t/plugin/ai-proxy-timeout-callsite.t +++ /dev/null @@ -1,136 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -use t::APISIX 'no_plan'; - -repeat_each(1); -no_long_string(); -no_root_location(); - -run_tests; - -__DATA__ - -=== TEST 1: ai-proxy resolves phase timeout fallbacks at the transport call site ---- config - location /t { - content_by_lua_block { - local provider_name = "apisix.plugins.ai-providers.timeout-capture" - local transport_name = "apisix.plugins.ai-transport.http" - local exporter_name = "apisix.plugins.prometheus.exporter" - local sanitize_name = "apisix.utils.log-sanitize" - local base_name = "apisix.plugins.ai-proxy.base" - - local saved = {} - for _, name in ipairs({ - provider_name, - transport_name, - exporter_name, - sanitize_name, - base_name, - }) do - saved[name] = package.loaded[name] - end - - local observed - package.loaded[provider_name] = { - capabilities = { - ["openai-chat"] = { - path = "/v1/chat/completions", - host = "127.0.0.1", - }, - }, - build_body = function(_, body) - return body, true - end, - build_request = function(_, _, body) - return { - method = "POST", - host = "127.0.0.1", - port = 80, - path = "/v1/chat/completions", - headers = {}, - body = body, - } - end, - } - package.loaded[transport_name] = { - request = function(_, timeout) - observed = timeout - return nil, "captured timeout" - end, - handle_error = function() - return 504 - end, - } - package.loaded[exporter_name] = { - inc_llm_active_connections = function() end, - } - package.loaded[sanitize_name] = { - redact_params = function(params) - return params - end, - } - package.loaded[base_name] = nil - - local base = require(base_name) - local cases = { - {name = "omitted", conf = {timeout = 900}}, - {name = "connect", conf = {timeout = 900, connect_timeout = 101}}, - {name = "send", conf = {timeout = 900, send_timeout = 202}}, - {name = "read", conf = {timeout = 900, read_timeout = 303}}, - } - - for _, case in ipairs(cases) do - observed = nil - local ctx = { - ai_client_protocol = "openai-chat", - picked_ai_instance = { - name = "capture", - provider = "timeout-capture", - options = {model = "gpt-4"}, - }, - var = {uri = "/t"}, - } - - ngx.ctx.api_ctx = ctx - base.before_proxy(case.conf, ctx) - if type(observed) == "number" then - ngx.say(case.name, ":number:", observed) - else - ngx.say(case.name, ":table:", - observed.connect_timeout, ",", - observed.send_timeout, ",", - observed.read_timeout) - end - end - ngx.ctx.api_ctx = nil - - for name, value in pairs(saved) do - package.loaded[name] = value - end - } - } ---- request -POST /t -{"model":"gpt-4","messages":[{"role":"user","content":"hello"}]} ---- more_headers -Content-Type: application/json ---- response_body -omitted:number:900 -connect:table:101,900,900 -send:table:900,202,900 -read:table:900,900,303