feat(ai-proxy): support phase-specific HTTP timeouts

apisix/plugins/ai-proxy/base.lua

@@ -341,7 +341,16 @@ function _M.before_proxy(conf, ctx, on_error)
                           core.json.delay_encode(log_sanitize.redact_params(params), true))
 
             -- Step 4: Send via transport
-            local res, transport_err, err_meta = transport_http.request(params, conf.timeout)
+            local timeout = conf.timeout
+            if conf.connect_timeout or conf.send_timeout or conf.read_timeout then
+                timeout = {
+                    connect_timeout = conf.connect_timeout or conf.timeout,
+                    send_timeout = conf.send_timeout or conf.timeout,
+                    read_timeout = conf.read_timeout or conf.timeout,
+                }
+            end
+
+            local res, transport_err, err_meta = transport_http.request(params, timeout)
             if not res then
                 core.log.warn("failed to send request to LLM server: ", transport_err)
                 if err_meta then

apisix/plugins/ai-proxy/schema.lua

@@ -344,6 +344,21 @@ _M.ai_proxy_schema = {
             default = 30000,
             description = "timeout in milliseconds",
         },
+        connect_timeout = {
+            type = "integer",
+            minimum = 1,
+            maximum = 600000,
+        },
+        send_timeout = {
+            type = "integer",
+            minimum = 1,
+            maximum = 600000,
+        },
+        read_timeout = {
+            type = "integer",
+            minimum = 1,
+            maximum = 600000,
+        },
         max_req_body_size = {
             type = "integer",
             minimum = 1,
@@ -478,6 +493,21 @@ _M.ai_proxy_multi_schema = {
             default = 30000,
             description = "timeout in milliseconds",
         },
+        connect_timeout = {
+            type = "integer",
+            minimum = 1,
+            maximum = 600000,
+        },
+        send_timeout = {
+            type = "integer",
+            minimum = 1,
+            maximum = 600000,
+        },
+        read_timeout = {
+            type = "integer",
+            minimum = 1,
+            maximum = 600000,
+        },
         max_req_body_size = {
             type = "integer",
             minimum = 1,

apisix/plugins/ai-transport/http.lua

@@ -94,7 +94,8 @@ end
 -- @param params table HTTP request parameters:
 --   {method, scheme, host, port, path, headers, query, body (table),
 --    ssl_verify, ssl_server_name}
--- @param timeout number Request timeout in milliseconds
+-- @param timeout number|table Request timeout in milliseconds, or a table with
+--   {connect_timeout, send_timeout, read_timeout} in milliseconds
 -- @return table|nil Response object (with body_reader, headers, status,
 --   _upstream_addr, _upstream_uri, _connect_time, _header_time, _t0)
 -- @return string|nil Error message
@@ -104,7 +105,12 @@ function _M.request(params, timeout)
     if not httpc then
         return nil, "failed to create http client: " .. (err or "unknown")
     end
-    httpc:set_timeout(timeout)
+    if type(timeout) == "table" then
+        httpc:set_timeouts(timeout.connect_timeout, timeout.send_timeout,
+                           timeout.read_timeout)
+    else
+        httpc:set_timeout(timeout)
+    end
 
     local upstream_addr = (params.host or "") .. ":" .. (params.port or "")
     local upstream_host = params.host or ""

docs/en/latest/plugins/ai-proxy-multi.md

@@ -137,7 +137,10 @@ When an instance's `provider` is set to `bedrock`, the Plugin expects requests i
 | semantic_opts.embeddings.auth.query | object         | False    |                                   |              | Authentication query parameters. Encrypted at rest. |
 | semantic_opts.embeddings.timeout    | integer        | False    | 3000                            | minimum = 1  | Embedding request timeout in milliseconds. The query prompt is embedded synchronously on every request, so this bounds the latency added to each request when the embedding endpoint is slow or down (the request then fails open to the fallback). |
 | semantic_opts.embeddings.ssl_verify | boolean        | False    | true                            |              | If true, verify the embedding service's certificate. |
-| timeout                             | integer        | False    | 30000                           | greater than or equal to 1 | Request timeout in milliseconds when requesting the LLM service. Applied per socket operation (connect / send / read block); does not cap the total duration of a streaming response. |
+| timeout                             | integer        | False    | 30000                           | 1 - 600000 | Default request timeout in milliseconds when requesting the LLM service. It remains the fallback for each phase below and is used for all phases when no phase-specific timeout is configured. |
+| connect_timeout                     | integer        | False    |                                 | 1 - 600000 | Connect timeout in milliseconds. Falls back to `timeout` when unset. |
+| send_timeout                        | integer        | False    |                                 | 1 - 600000 | Send timeout in milliseconds. Falls back to `timeout` when unset. |
+| read_timeout                        | integer        | False    |                                 | 1 - 600000 | Read timeout in milliseconds. Falls back to `timeout` when unset. This applies per socket read and does not cap the total duration of a streaming response. |
 | max_req_body_size                   | integer        | False    | 67108864                        | greater than or equal to 1 | Maximum request body size in bytes that the plugin reads into memory. Requests whose body exceeds this limit are rejected with `413`. Prevents unbounded memory buffering of large request bodies. |
 | max_stream_duration_ms              | integer        | False    |                                 | greater than or equal to 1 | Maximum wall-clock duration (in milliseconds) for a streaming AI response. If the upstream keeps sending data past this deadline, the gateway closes the connection. Unset means no cap. Use this to protect the gateway from upstream bugs that produce tokens indefinitely. When the limit is hit mid-stream, the downstream SSE stream is truncated (no protocol-specific terminator such as `[DONE]`, `message_stop`, or `response.completed`); well-behaved clients should treat a missing terminator as an incomplete response. |
 | max_response_bytes                  | integer        | False    |                                 | greater than or equal to 1 | Maximum total bytes read from the upstream for a single AI response (streaming or non-streaming). If exceeded, the gateway closes the connection. For non-streaming responses with `Content-Length`, the check is performed before reading the body; for chunked (no-`Content-Length`) non-streaming responses and for streaming responses, the cap is enforced incrementally as bytes are received. Unset means no cap. |

docs/en/latest/plugins/ai-proxy.md

@@ -93,7 +93,10 @@ When `provider` is set to `bedrock`, the Plugin expects requests in the [Bedrock
 | logging        | object  | False    |         |                                          | Logging configurations. Does not affect `error.log`. |
 | logging.summaries | boolean | False | false |                                          | If true, logs request LLM model, duration, request, and response tokens. |
 | logging.payloads  | boolean | False | false |                                          | If true, logs request and response payload. |
-| timeout        | integer | False    | 30000    | 1 - 600000                               | Request timeout in milliseconds when requesting the LLM service. Applied per socket operation (connect / send / read block); does not cap the total duration of a streaming response. |
+| timeout        | integer | False    | 30000    | 1 - 600000                               | Default request timeout in milliseconds when requesting the LLM service. It remains the fallback for each phase below and is used for all phases when no phase-specific timeout is configured. |
+| connect_timeout | integer | False   |          | 1 - 600000                               | Connect timeout in milliseconds. Falls back to `timeout` when unset. |
+| send_timeout   | integer | False    |          | 1 - 600000                               | Send timeout in milliseconds. Falls back to `timeout` when unset. |
+| read_timeout   | integer | False    |          | 1 - 600000                               | Read timeout in milliseconds. Falls back to `timeout` when unset. This applies per socket read and does not cap the total duration of a streaming response. |
 | max_stream_duration_ms | integer | False |        | ≥ 1                                      | Maximum wall-clock duration (in milliseconds) for a streaming AI response. If the upstream keeps sending data past this deadline, the gateway closes the connection. Unset means no cap. Use this to protect the gateway from upstream bugs that produce tokens indefinitely. When the limit is hit mid-stream, the downstream SSE stream is truncated (no protocol-specific terminator such as `[DONE]`, `message_stop`, or `response.completed`); well-behaved clients should treat a missing terminator as an incomplete response. |
 | max_response_bytes     | integer | False |        | ≥ 1                                      | Maximum total bytes read from the upstream for a single AI response (streaming or non-streaming). If exceeded, the gateway closes the connection. For non-streaming responses with `Content-Length`, the check is performed before reading the body; for chunked (no-`Content-Length`) non-streaming responses and for streaming responses, the cap is enforced incrementally as bytes are received. Unset means no cap. |
 | max_req_body_size | integer | False | 67108864 | >= 1 | Maximum request body size in bytes that the plugin reads into memory. Requests whose body exceeds this limit are rejected with `413`. Prevents unbounded memory buffering of large request bodies. |

docs/zh/latest/plugins/ai-proxy-multi.md

@@ -139,7 +139,10 @@ import TabItem from '@theme/TabItem';
 | semantic_opts.embeddings.auth.query | object         | 否    |                                   |              | 认证查询参数。加密存储。 |
 | semantic_opts.embeddings.timeout    | integer        | 否    | 3000                            | 最小值为 1  | 嵌入请求的超时时间（毫秒）。每个请求都会同步嵌入查询提示词，因此该值限定了当嵌入端点变慢或不可用时每个请求增加的延迟上限（随后请求会 fail-open 到回退实例）。 |
 | semantic_opts.embeddings.ssl_verify | boolean        | 否    | true                            |              | 如果为 true，验证嵌入服务的证书。 |
-| timeout                             | integer        | 否    | 30000                           | 大于或等于 1 | 请求 LLM 服务时的请求超时时间（毫秒）。应用于单次 socket 操作（连接 / 发送 / 读取块），不限制流式响应的总时长。 |
+| timeout                             | integer        | 否    | 30000                           | 1 - 600000 | 请求 LLM 服务时的默认超时时间（毫秒）。它是下列各阶段的回退值；未配置任一阶段专用超时时间时，所有阶段均使用该值。 |
+| connect_timeout                     | integer        | 否    |                                 | 1 - 600000 | 连接超时时间（毫秒）。未设置时回退到 `timeout`。 |
+| send_timeout                        | integer        | 否    |                                 | 1 - 600000 | 发送超时时间（毫秒）。未设置时回退到 `timeout`。 |
+| read_timeout                        | integer        | 否    |                                 | 1 - 600000 | 读取超时时间（毫秒）。未设置时回退到 `timeout`。它针对每次 socket 读取，不限制流式响应的总时长。 |
 | max_stream_duration_ms              | integer        | 否    |                                 | 大于或等于 1 | 流式 AI 响应的总墙钟时长上限（毫秒）。若上游在此时间后仍持续发送数据，网关将关闭连接。未设置时不限制。用于防护上游持续输出 token 导致网关 CPU 被打满的异常情况。中途触发上限时，下游 SSE 流会被截断（不再发送协议特定的终止标记，例如 `[DONE]`、`message_stop` 或 `response.completed`），客户端应将缺失的终止标记视为响应未完成。 |
 | max_response_bytes                  | integer        | 否    |                                 | 大于或等于 1 | 单次 AI 响应（流式或非流式）允许从上游读取的最大总字节数。超出时关闭连接。非流式响应若存在 `Content-Length`，在读取 body 之前预检；否则（chunked 传输）与流式响应一样在接收字节的过程中增量检查。未设置时不限制。 |
 | keepalive                           | boolean        | 否    | true                            |              | 如果为 true，在请求 LLM 服务时保持连接活跃。 |

docs/zh/latest/plugins/ai-proxy.md

@@ -93,7 +93,10 @@ import TabItem from '@theme/TabItem';
 | logging        | object  | 否    |         |                                          | 日志配置。不影响 `error.log`。 |
 | logging.summaries | boolean | 否 | false |                                          | 如果为 true，记录请求 LLM 模型、持续时间、请求和响应令牌。 |
 | logging.payloads  | boolean | 否 | false |                                          | 如果为 true，记录请求和响应负载。 |
-| timeout        | integer | 否    | 30000    | 1 - 600000                               | 请求 LLM 服务时的请求超时时间（毫秒）。按单次 socket 操作（连接 / 发送 / 读取数据块）计算，不限制流式响应的总时长。 |
+| timeout        | integer | 否    | 30000    | 1 - 600000                               | 请求 LLM 服务时的默认超时时间（毫秒）。它是下列各阶段的回退值；未配置任一阶段专用超时时间时，所有阶段均使用该值。 |
+| connect_timeout | integer | 否   |          | 1 - 600000                               | 连接超时时间（毫秒）。未设置时回退到 `timeout`。 |
+| send_timeout   | integer | 否    |          | 1 - 600000                               | 发送超时时间（毫秒）。未设置时回退到 `timeout`。 |
+| read_timeout   | integer | 否    |          | 1 - 600000                               | 读取超时时间（毫秒）。未设置时回退到 `timeout`。它针对每次 socket 读取，不限制流式响应的总时长。 |
 | max_stream_duration_ms | integer | 否 |        | ≥ 1                                      | 流式 AI 响应的最大墙钟时长（毫秒）。如果上游在该截止时间后仍持续发送数据，网关会关闭连接。不设置表示不限制。用于防止上游异常无限产出 token。当在流式过程中触发该限制时，下游 SSE 流会被截断（不会发送 `[DONE]`、`message_stop`、`response.completed` 等协议终止标记）；行为正常的客户端应将缺少终止标记视为不完整的响应。 |
 | max_response_bytes     | integer | 否 |        | ≥ 1                                      | 单次 AI 响应（流式或非流式）从上游读取的最大总字节数。超过则网关关闭连接。对于带 `Content-Length` 的非流式响应，在读取响应体前进行检查；对于分块（无 `Content-Length`）的非流式响应以及流式响应，则在接收字节的过程中增量地强制执行该上限。不设置表示不限制。 |
 | keepalive      | boolean | 否    | true   |                                          | 如果为 true，在请求 LLM 服务时保持连接活跃。 |

t/plugin/ai-proxy-multi.t

@@ -77,6 +77,73 @@ passed
 
 
 
+=== TEST 1a: phase timeouts accept valid values and reject invalid values
+--- config
+    location /t {
+        content_by_lua_block {
+            local plugin = require("apisix.plugins.ai-proxy-multi")
+
+            local function base_conf()
+                return {
+                    instances = {
+                        {
+                            name = "openai-official",
+                            provider = "openai",
+                            options = {model = "gpt-4"},
+                            weight = 1,
+                            auth = {header = {some_header = "some_value"}},
+                        }
+                    }
+                }
+            end
+
+            local valid_cases = {
+                {},
+                {connect_timeout = 1, send_timeout = 1, read_timeout = 1},
+                {connect_timeout = 600000, send_timeout = 600000, read_timeout = 600000},
+                {connect_timeout = 101, send_timeout = 202, read_timeout = 303},
+            }
+            for _, timeouts in ipairs(valid_cases) do
+                local conf = base_conf()
+                for key, value in pairs(timeouts) do
+                    conf[key] = value
+                end
+                assert(plugin.check_schema(conf))
+                ngx.say("valid")
+            end
+
+            local invalid_values = {0, 600001, 1.5, "invalid"}
+            for _, key in ipairs({"connect_timeout", "send_timeout", "read_timeout"}) do
+                for _, value in ipairs(invalid_values) do
+                    local conf = base_conf()
+                    conf[key] = value
+                    local ok = plugin.check_schema(conf)
+                    assert(not ok)
+                    ngx.say("invalid: ", key)
+                end
+            end
+        }
+    }
+--- response_body
+valid
+valid
+valid
+valid
+invalid: connect_timeout
+invalid: connect_timeout
+invalid: connect_timeout
+invalid: connect_timeout
+invalid: send_timeout
+invalid: send_timeout
+invalid: send_timeout
+invalid: send_timeout
+invalid: read_timeout
+invalid: read_timeout
+invalid: read_timeout
+invalid: read_timeout
+
+
+
 === TEST 2: unsupported provider
 --- config
     location /t {