From 870b0a06421e50fe938787c8d4638f3d09a76cc6 Mon Sep 17 00:00:00 2001 From: slytechnical Date: Tue, 27 May 2025 12:44:33 -0500 Subject: [PATCH] Added a hardcoded list of computer use models for litellm as a fallback for older litellm versions --- .../fetchers/__tests__/litellm.test.ts | 199 ++++++++++++++++++ src/api/providers/fetchers/litellm.ts | 17 +- src/shared/api.ts | 33 +++ 3 files changed, 247 insertions(+), 2 deletions(-) diff --git a/src/api/providers/fetchers/__tests__/litellm.test.ts b/src/api/providers/fetchers/__tests__/litellm.test.ts index 4e474cd9956..49e928548ff 100644 --- a/src/api/providers/fetchers/__tests__/litellm.test.ts +++ b/src/api/providers/fetchers/__tests__/litellm.test.ts @@ -248,4 +248,203 @@ describe("getLiteLLMModels", () => { expect(result).toEqual({}) }) + + it("uses fallback computer use detection when supports_computer_use is not available", async () => { + const mockResponse = { + data: { + data: [ + { + model_name: "claude-3-5-sonnet-latest", + model_info: { + max_tokens: 4096, + max_input_tokens: 200000, + supports_vision: true, + supports_prompt_caching: false, + // Note: no supports_computer_use field + }, + litellm_params: { + model: "anthropic/claude-3-5-sonnet-latest", // This should match the fallback list + }, + }, + { + model_name: "gpt-4-turbo", + model_info: { + max_tokens: 8192, + max_input_tokens: 128000, + supports_vision: false, + supports_prompt_caching: false, + // Note: no supports_computer_use field + }, + litellm_params: { + model: "openai/gpt-4-turbo", // This should NOT match the fallback list + }, + }, + ], + }, + } + + mockedAxios.get.mockResolvedValue(mockResponse) + + const result = await getLiteLLMModels("test-api-key", "http://localhost:4000") + + expect(result["claude-3-5-sonnet-latest"]).toEqual({ + maxTokens: 4096, + contextWindow: 200000, + supportsImages: true, + supportsComputerUse: true, // Should be true due to fallback + supportsPromptCache: false, + inputPrice: undefined, + outputPrice: undefined, + description: "claude-3-5-sonnet-latest via LiteLLM proxy", + }) + + expect(result["gpt-4-turbo"]).toEqual({ + maxTokens: 8192, + contextWindow: 128000, + supportsImages: false, + supportsComputerUse: false, // Should be false as it's not in fallback list + supportsPromptCache: false, + inputPrice: undefined, + outputPrice: undefined, + description: "gpt-4-turbo via LiteLLM proxy", + }) + }) + + it("prioritizes explicit supports_computer_use over fallback detection", async () => { + const mockResponse = { + data: { + data: [ + { + model_name: "claude-3-5-sonnet-latest", + model_info: { + max_tokens: 4096, + max_input_tokens: 200000, + supports_vision: true, + supports_prompt_caching: false, + supports_computer_use: false, // Explicitly set to false + }, + litellm_params: { + model: "anthropic/claude-3-5-sonnet-latest", // This matches fallback list but should be ignored + }, + }, + { + model_name: "custom-model", + model_info: { + max_tokens: 8192, + max_input_tokens: 128000, + supports_vision: false, + supports_prompt_caching: false, + supports_computer_use: true, // Explicitly set to true + }, + litellm_params: { + model: "custom/custom-model", // This would NOT match fallback list + }, + }, + { + model_name: "another-custom-model", + model_info: { + max_tokens: 8192, + max_input_tokens: 128000, + supports_vision: false, + supports_prompt_caching: false, + supports_computer_use: false, // Explicitly set to false + }, + litellm_params: { + model: "custom/another-custom-model", // This would NOT match fallback list + }, + }, + ], + }, + } + + mockedAxios.get.mockResolvedValue(mockResponse) + + const result = await getLiteLLMModels("test-api-key", "http://localhost:4000") + + expect(result["claude-3-5-sonnet-latest"]).toEqual({ + maxTokens: 4096, + contextWindow: 200000, + supportsImages: true, + supportsComputerUse: false, // False because explicitly set to false (fallback ignored) + supportsPromptCache: false, + inputPrice: undefined, + outputPrice: undefined, + description: "claude-3-5-sonnet-latest via LiteLLM proxy", + }) + + expect(result["custom-model"]).toEqual({ + maxTokens: 8192, + contextWindow: 128000, + supportsImages: false, + supportsComputerUse: true, // True because explicitly set to true + supportsPromptCache: false, + inputPrice: undefined, + outputPrice: undefined, + description: "custom-model via LiteLLM proxy", + }) + + expect(result["another-custom-model"]).toEqual({ + maxTokens: 8192, + contextWindow: 128000, + supportsImages: false, + supportsComputerUse: false, // False because explicitly set to false + supportsPromptCache: false, + inputPrice: undefined, + outputPrice: undefined, + description: "another-custom-model via LiteLLM proxy", + }) + }) + + it("handles fallback detection with various model name formats", async () => { + const mockResponse = { + data: { + data: [ + { + model_name: "vertex-claude", + model_info: { + max_tokens: 4096, + max_input_tokens: 200000, + supports_vision: true, + supports_prompt_caching: false, + }, + litellm_params: { + model: "vertex_ai/claude-3-5-sonnet", // Should match fallback list + }, + }, + { + model_name: "openrouter-claude", + model_info: { + max_tokens: 4096, + max_input_tokens: 200000, + supports_vision: true, + supports_prompt_caching: false, + }, + litellm_params: { + model: "openrouter/anthropic/claude-3.5-sonnet", // Should match fallback list + }, + }, + { + model_name: "bedrock-claude", + model_info: { + max_tokens: 4096, + max_input_tokens: 200000, + supports_vision: true, + supports_prompt_caching: false, + }, + litellm_params: { + model: "anthropic.claude-3-5-sonnet-20241022-v2:0", // Should match fallback list + }, + }, + ], + }, + } + + mockedAxios.get.mockResolvedValue(mockResponse) + + const result = await getLiteLLMModels("test-api-key", "http://localhost:4000") + + expect(result["vertex-claude"].supportsComputerUse).toBe(true) + expect(result["openrouter-claude"].supportsComputerUse).toBe(true) + expect(result["bedrock-claude"].supportsComputerUse).toBe(true) + }) }) diff --git a/src/api/providers/fetchers/litellm.ts b/src/api/providers/fetchers/litellm.ts index dbd6ccc73a8..093fd85888f 100644 --- a/src/api/providers/fetchers/litellm.ts +++ b/src/api/providers/fetchers/litellm.ts @@ -1,6 +1,6 @@ import axios from "axios" -import { ModelRecord } from "../../../shared/api" +import { LITELLM_COMPUTER_USE_MODELS, ModelRecord } from "../../../shared/api" /** * Fetches available models from a LiteLLM server @@ -23,6 +23,8 @@ export async function getLiteLLMModels(apiKey: string, baseUrl: string): Promise const response = await axios.get(`${baseUrl}/v1/model/info`, { headers, timeout: 5000 }) const models: ModelRecord = {} + const computerModels = Array.from(LITELLM_COMPUTER_USE_MODELS) + // Process the model info from the response if (response.data && response.data.data && Array.isArray(response.data.data)) { for (const model of response.data.data) { @@ -32,12 +34,23 @@ export async function getLiteLLMModels(apiKey: string, baseUrl: string): Promise if (!modelName || !modelInfo || !litellmModelName) continue + // Use explicit supports_computer_use if available, otherwise fall back to hardcoded list + let supportsComputerUse: boolean + if (modelInfo.supports_computer_use !== undefined) { + supportsComputerUse = Boolean(modelInfo.supports_computer_use) + } else { + // Fallback for older LiteLLM versions that don't have supports_computer_use field + supportsComputerUse = computerModels.some((computer_model) => + litellmModelName.endsWith(computer_model), + ) + } + models[modelName] = { maxTokens: modelInfo.max_tokens || 8192, contextWindow: modelInfo.max_input_tokens || 200000, supportsImages: Boolean(modelInfo.supports_vision), // litellm_params.model may have a prefix like openrouter/ - supportsComputerUse: Boolean(modelInfo.supports_computer_use), + supportsComputerUse, supportsPromptCache: Boolean(modelInfo.supports_prompt_caching), inputPrice: modelInfo.input_cost_per_token ? modelInfo.input_cost_per_token * 1000000 : undefined, outputPrice: modelInfo.output_cost_per_token diff --git a/src/shared/api.ts b/src/shared/api.ts index 48c397d1b42..76d7b185df3 100644 --- a/src/shared/api.ts +++ b/src/shared/api.ts @@ -1260,6 +1260,39 @@ export const litellmDefaultModelInfo: ModelInfo = { cacheWritesPrice: 3.75, cacheReadsPrice: 0.3, } + +export const LITELLM_COMPUTER_USE_MODELS = new Set([ + "claude-3-5-sonnet-latest", + "claude-opus-4-20250514", + "claude-sonnet-4-20250514", + "claude-3-7-sonnet-latest", + "claude-3-7-sonnet-20250219", + "claude-3-5-sonnet-20241022", + "vertex_ai/claude-3-5-sonnet", + "vertex_ai/claude-3-5-sonnet-v2", + "vertex_ai/claude-3-5-sonnet-v2@20241022", + "vertex_ai/claude-3-7-sonnet@20250219", + "vertex_ai/claude-opus-4@20250514", + "vertex_ai/claude-sonnet-4@20250514", + "openrouter/anthropic/claude-3.5-sonnet", + "openrouter/anthropic/claude-3.5-sonnet:beta", + "openrouter/anthropic/claude-3.7-sonnet", + "openrouter/anthropic/claude-3.7-sonnet:beta", + "anthropic.claude-opus-4-20250514-v1:0", + "anthropic.claude-sonnet-4-20250514-v1:0", + "anthropic.claude-3-7-sonnet-20250219-v1:0", + "anthropic.claude-3-5-sonnet-20241022-v2:0", + "us.anthropic.claude-3-5-sonnet-20241022-v2:0", + "us.anthropic.claude-3-7-sonnet-20250219-v1:0", + "us.anthropic.claude-opus-4-20250514-v1:0", + "us.anthropic.claude-sonnet-4-20250514-v1:0", + "eu.anthropic.claude-3-5-sonnet-20241022-v2:0", + "eu.anthropic.claude-3-7-sonnet-20250219-v1:0", + "eu.anthropic.claude-opus-4-20250514-v1:0", + "eu.anthropic.claude-sonnet-4-20250514-v1:0", + "snowflake/claude-3-5-sonnet", +]) + // xAI // https://docs.x.ai/docs/api-reference export type XAIModelId = keyof typeof xaiModels