From a7a127e93518b4b91820d23b4b508a70f9834a98 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Wed, 29 Jul 2026 18:14:19 +0800 Subject: [PATCH] Add optional image URL base64 compatibility --- .env.example | 2 + relay/channel/openai/image_url_compat.go | 162 ++++++++++++++++++ relay/channel/openai/image_url_compat_test.go | 70 ++++++++ relay/channel/openai/relay_image.go | 10 ++ 4 files changed, 244 insertions(+) create mode 100644 relay/channel/openai/image_url_compat.go create mode 100644 relay/channel/openai/image_url_compat_test.go diff --git a/.env.example b/.env.example index d62e114..76ea82f 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,7 @@ # 端口号 # PORT=3000 +# Convert URL-based OpenAI image responses to b64_json for strict clients such as Codex. +# CODEX_IMAGE_URL_TO_B64=false # 前端基础URL # FRONTEND_BASE_URL=https://your-frontend-url.com diff --git a/relay/channel/openai/image_url_compat.go b/relay/channel/openai/image_url_compat.go new file mode 100644 index 0000000..e0f222c --- /dev/null +++ b/relay/channel/openai/image_url_compat.go @@ -0,0 +1,162 @@ +package openai + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/QuantumNous/new-api/common" +) + +const maxImageURLCompatBytes int64 = 40 * 1024 * 1024 + +type imageURLCompatDownloadFunc func(string, ...string) (*http.Response, error) + +type imageURLCompatStats struct { + Converted int + DownloadedBytes int64 + DownloadTime time.Duration + EncodeTime time.Duration + TotalTime time.Duration +} + +func (s imageURLCompatStats) ServerTiming() string { + return fmt.Sprintf( + "image-url-download;dur=%.3f, image-b64-encode;dur=%.3f, image-url-compat;dur=%.3f", + float64(s.DownloadTime.Microseconds())/1000, + float64(s.EncodeTime.Microseconds())/1000, + float64(s.TotalTime.Microseconds())/1000, + ) +} + +func (s imageURLCompatStats) LogMessage() string { + return fmt.Sprintf( + "image URL compatibility converted=%d bytes=%d download_ms=%.3f encode_ms=%.3f total_ms=%.3f", + s.Converted, + s.DownloadedBytes, + float64(s.DownloadTime.Microseconds())/1000, + float64(s.EncodeTime.Microseconds())/1000, + float64(s.TotalTime.Microseconds())/1000, + ) +} + +func convertOpenAIImageURLsToBase64(responseBody []byte, download imageURLCompatDownloadFunc) ([]byte, imageURLCompatStats, error) { + var stats imageURLCompatStats + if !common.GetEnvOrDefaultBool("CODEX_IMAGE_URL_TO_B64", false) { + return responseBody, stats, nil + } + + startedAt := time.Now() + var payload map[string]json.RawMessage + if err := common.Unmarshal(responseBody, &payload); err != nil { + return nil, stats, fmt.Errorf("decode image compatibility response: %w", err) + } + dataJSON, ok := payload["data"] + if !ok { + return responseBody, stats, nil + } + + var images []map[string]json.RawMessage + if err := common.Unmarshal(dataJSON, &images); err != nil { + return nil, stats, fmt.Errorf("decode image compatibility data: %w", err) + } + + for _, image := range images { + if jsonString(image["b64_json"]) != "" { + continue + } + + var imageURL string + var sourceField string + for _, field := range []string{"url", "result_url", "image_url"} { + if value := jsonString(image[field]); value != "" { + imageURL = value + sourceField = field + break + } + } + if imageURL == "" { + continue + } + + downloadStartedAt := time.Now() + imageResponse, err := download(imageURL, "OpenAI image URL compatibility") + if err != nil { + return nil, stats, fmt.Errorf("download image compatibility URL: %w", err) + } + imageBytes, readErr := readImageURLCompatResponse(imageResponse) + stats.DownloadTime += time.Since(downloadStartedAt) + if readErr != nil { + return nil, stats, readErr + } + + encodeStartedAt := time.Now() + encodedJSON, err := common.Marshal(base64.StdEncoding.EncodeToString(imageBytes)) + if err != nil { + return nil, stats, fmt.Errorf("encode image compatibility response: %w", err) + } + image["b64_json"] = encodedJSON + delete(image, sourceField) + stats.EncodeTime += time.Since(encodeStartedAt) + stats.Converted++ + stats.DownloadedBytes += int64(len(imageBytes)) + } + + if stats.Converted == 0 { + return responseBody, stats, nil + } + + encodeStartedAt := time.Now() + dataJSON, err := common.Marshal(images) + if err != nil { + return nil, stats, fmt.Errorf("encode image compatibility data: %w", err) + } + payload["data"] = dataJSON + convertedBody, err := common.Marshal(payload) + stats.EncodeTime += time.Since(encodeStartedAt) + stats.TotalTime = time.Since(startedAt) + if err != nil { + return nil, stats, fmt.Errorf("encode image compatibility payload: %w", err) + } + return convertedBody, stats, nil +} + +func jsonString(raw json.RawMessage) string { + if len(raw) == 0 { + return "" + } + var value string + if err := common.Unmarshal(raw, &value); err != nil { + return "" + } + return strings.TrimSpace(value) +} + +func readImageURLCompatResponse(response *http.Response) ([]byte, error) { + if response == nil || response.Body == nil { + return nil, fmt.Errorf("image compatibility download returned an empty response") + } + defer response.Body.Close() + if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices { + return nil, fmt.Errorf("image compatibility download returned HTTP %d", response.StatusCode) + } + if response.ContentLength > maxImageURLCompatBytes { + return nil, fmt.Errorf("image compatibility download exceeds %d bytes", maxImageURLCompatBytes) + } + + imageBytes, err := io.ReadAll(io.LimitReader(response.Body, maxImageURLCompatBytes+1)) + if err != nil { + return nil, fmt.Errorf("read image compatibility download: %w", err) + } + if int64(len(imageBytes)) > maxImageURLCompatBytes { + return nil, fmt.Errorf("image compatibility download exceeds %d bytes", maxImageURLCompatBytes) + } + if len(imageBytes) == 0 { + return nil, fmt.Errorf("image compatibility download returned an empty body") + } + return imageBytes, nil +} diff --git a/relay/channel/openai/image_url_compat_test.go b/relay/channel/openai/image_url_compat_test.go new file mode 100644 index 0000000..2722bba --- /dev/null +++ b/relay/channel/openai/image_url_compat_test.go @@ -0,0 +1,70 @@ +package openai + +import ( + "bytes" + "io" + "net/http" + "testing" + + "github.com/QuantumNous/new-api/common" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConvertOpenAIImageURLsToBase64Disabled(t *testing.T) { + t.Setenv("CODEX_IMAGE_URL_TO_B64", "false") + body := []byte(`{"created":1,"data":[{"url":"https://example.com/image.png"}]}`) + + converted, stats, err := convertOpenAIImageURLsToBase64(body, func(string, ...string) (*http.Response, error) { + t.Fatal("download must not run while compatibility is disabled") + return nil, nil + }) + + require.NoError(t, err) + assert.Equal(t, body, converted) + assert.Zero(t, stats.Converted) +} + +func TestConvertOpenAIImageURLsToBase64PreservesResponseFields(t *testing.T) { + t.Setenv("CODEX_IMAGE_URL_TO_B64", "true") + image := []byte("test-image") + body := []byte(`{"created":1,"data":[{"url":"https://example.com/image.png","revised_prompt":"kept"}],"usage":{"total_tokens":7},"custom":{"kept":true}}`) + + converted, stats, err := convertOpenAIImageURLsToBase64(body, func(rawURL string, reason ...string) (*http.Response, error) { + assert.Equal(t, "https://example.com/image.png", rawURL) + assert.Equal(t, []string{"OpenAI image URL compatibility"}, reason) + return &http.Response{ + StatusCode: http.StatusOK, + ContentLength: int64(len(image)), + Body: io.NopCloser(bytes.NewReader(image)), + }, nil + }) + + require.NoError(t, err) + assert.Equal(t, 1, stats.Converted) + assert.Equal(t, int64(len(image)), stats.DownloadedBytes) + + var payload map[string]any + require.NoError(t, common.Unmarshal(converted, &payload)) + data := payload["data"].([]any) + item := data[0].(map[string]any) + assert.Equal(t, "dGVzdC1pbWFnZQ==", item["b64_json"]) + assert.Equal(t, "kept", item["revised_prompt"]) + assert.NotContains(t, item, "url") + assert.Equal(t, float64(7), payload["usage"].(map[string]any)["total_tokens"]) + assert.Equal(t, true, payload["custom"].(map[string]any)["kept"]) + assert.Contains(t, stats.ServerTiming(), "image-url-compat") +} + +func TestReadImageURLCompatResponseRejectsOversize(t *testing.T) { + response := &http.Response{ + StatusCode: http.StatusOK, + ContentLength: maxImageURLCompatBytes + 1, + Body: io.NopCloser(bytes.NewReader([]byte("unused"))), + } + + _, err := readImageURLCompatResponse(response) + + require.Error(t, err) + assert.Contains(t, err.Error(), "exceeds") +} diff --git a/relay/channel/openai/relay_image.go b/relay/channel/openai/relay_image.go index e0f09aa..a6cfad1 100644 --- a/relay/channel/openai/relay_image.go +++ b/relay/channel/openai/relay_image.go @@ -39,6 +39,16 @@ func OpenaiImageHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http. return nil, types.NewOpenAIError(err, types.ErrorCodeReadResponseBodyFailed, http.StatusInternalServerError) } + responseBody, compatStats, err := convertOpenAIImageURLsToBase64(responseBody, service.DoDownloadRequest) + if err != nil { + return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusBadGateway) + } + if compatStats.Converted > 0 { + c.Header("Server-Timing", compatStats.ServerTiming()) + c.Header("X-New-API-Image-Compat", "url-to-b64") + logger.LogInfo(c, compatStats.LogMessage()) + } + var usageResp dto.SimpleResponse err = common.Unmarshal(responseBody, &usageResp) if err != nil {