obot-platform · cjellick · Feb 26, 2025 · Feb 26, 2025 · Feb 26, 2025
diff --git a/browser/package-lock.json b/browser/package-lock.json
diff --git a/browser/package.json b/browser/package.json
@@ -26,7 +26,7 @@
     "env-paths": "^3.0.0",
     "express": "^4.18.2",
     "global-cache-dir": "^6.0.0",
-    "playwright": "^1.48.1",
+    "playwright": "^1.50.1",
     "sharp": "^0.33.5",
     "ts-node": "^10.9.2",
     "ts-node-dev": "^2.0.0",

diff --git a/browser/src/browse.ts b/browser/src/browse.ts
@@ -4,6 +4,7 @@ import { GPTScript } from '@gptscript-ai/gptscript'
 import { delay } from './delay.ts'
 import { URL } from 'url'
 import TurndownService from 'turndown'
+import { ModelProviderCredentials } from './session.ts'
 
 export async function close (page: Page): Promise<void> {
   await page.close()
@@ -150,7 +151,7 @@ export async function filterContent (page: Page, tabID: string, printTabID: bool
 }
 
 // inspect inspects a webpage and returns a locator for a specific element based on the user's input and action.
-export async function inspect (page: Page, model: string, userInput: string, action: string, matchTextOnly: boolean, keywords?: string[]): Promise<string[]> {
+export async function inspect (page: Page, modelProviderCredentials: ModelProviderCredentials | undefined, model: string, userInput: string, action: string, matchTextOnly: boolean, keywords?: string[]): Promise<string[]> {
   if (userInput === '') {
     // This shouldn't happen, since the LLM is told that it must fill in the user input,
     // but in case it doesn't, just use the keywords.
@@ -223,13 +224,16 @@ export async function inspect (page: Page, model: string, userInput: string, act
   const run = await client.evaluate({
     modelName: model,
     instructions
+  }, {
+    BaseURL: modelProviderCredentials?.baseUrl,
+    APIKey: modelProviderCredentials?.apiKey
   })
 
   const output = (await run.text()).replace('\n', '').trim()
   return [output]
 }
 
-export async function inspectForSelect (page: Page, model: string, userInput: string, userSelection: string, keywords?: string[]): Promise<{ selector: string, option: string }> {
+export async function inspectForSelect (page: Page, modelProviderCredentials: ModelProviderCredentials | undefined, model: string, userInput: string, userSelection: string, keywords?: string[]): Promise<{ selector: string, option: string }> {
   let elementData = ''
   const modes = ['matchAll', 'oneSibling', 'twoSiblings', 'parent', 'grandparent']
   for (const mode of modes) {
@@ -267,6 +271,9 @@ export async function inspectForSelect (page: Page, model: string, userInput: st
   const run = await client.evaluate({
     modelName: model,
     instructions
+  }, {
+    BaseURL: modelProviderCredentials?.baseUrl,
+    APIKey: modelProviderCredentials?.apiKey
   })
 
   const output = (await run.text()).replace('\n', '').trim()

diff --git a/browser/src/download.ts b/browser/src/download.ts
@@ -34,7 +34,7 @@ export async function download(
     const contentType = response.headers['content-type']
     await client.writeFileInWorkspace(workspaceFile, Buffer.from(response.data), workspaceId)
 
-    return { 
+    return {
         url,
         resolvedUrl: response.request.res.responseUrl || url,
         contentType,

diff --git a/browser/src/fill.ts b/browser/src/fill.ts
@@ -1,9 +1,10 @@
 import { delay } from './delay.ts'
 import { type Page } from 'playwright'
 import { inspect } from './browse.ts'
+import { ModelProviderCredentials } from './session.ts'
 
-export async function fill (page: Page, model: string, userInput: string, content: string, keywords: string[], matchTextOnly: boolean): Promise<void> {
-  const locators = await inspect(page, model, userInput, 'fill', matchTextOnly, keywords)
+export async function fill (page: Page, modelProviderCredentials: ModelProviderCredentials | undefined, model: string, userInput: string, content: string, keywords: string[], matchTextOnly: boolean): Promise<void> {
+  const locators = await inspect(page, modelProviderCredentials, model, userInput, 'fill', matchTextOnly, keywords)
   for (const locator of locators) {
     console.log(locator)
     try {

diff --git a/browser/src/select.ts b/browser/src/select.ts
@@ -1,8 +1,9 @@
 import { type Page } from 'playwright'
 import { inspectForSelect } from './browse.ts'
+import { ModelProviderCredentials } from './session.ts'
 
-export async function select (page: Page, model: string, userInput: string, option: string): Promise<void> {
-  const selection = await inspectForSelect(page, model, option, userInput)
+export async function select (page: Page, modelProviderCredentials: ModelProviderCredentials | undefined, model: string, userInput: string, option: string): Promise<void> {
+  const selection = await inspectForSelect(page, modelProviderCredentials, model, option, userInput)
 
   try {
     await page.selectOption(`${selection.selector}`, selection.option, { timeout: 5000 })

diff --git a/browser/src/server.ts b/browser/src/server.ts
@@ -6,7 +6,7 @@ import { fill } from './fill.ts'
 import { enter } from './enter.ts'
 import { scrollToBottom } from './scrollToBottom.ts'
 import { randomBytes } from 'node:crypto'
-import { getSessionId, SessionManager } from './session.ts'
+import { getSessionId, SessionManager, getModelProviderCredentials } from './session.ts'
 import { screenshot, ScreenshotInfo } from './screenshot.ts'
 import { download } from './download.ts'
 
@@ -58,6 +58,7 @@ async function main (): Promise<void> {
     const keywords: string[] = (data.keywords ?? '').split(',')
     const filter: string = data.filter ?? ''
     const followMode: boolean = data.followMode === 'false' ? false : Boolean(data.followMode)
+    const modelProviderCredentials = getModelProviderCredentials(req.headers)
 
     try {
       const sessionID = getSessionId(req.headers)
@@ -100,6 +101,7 @@ async function main (): Promise<void> {
             // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
             response.result = await fill(
               page,
+              modelProviderCredentials,
               model,
               userInput,
               data.content ?? '',

diff --git a/browser/src/session.ts b/browser/src/session.ts
@@ -143,6 +143,21 @@ export function getWorkspaceId (headers: IncomingHttpHeaders): string | undefine
   return getGPTScriptEnv(headers, 'GPTSCRIPT_WORKSPACE_ID')
 }
 
+export interface ModelProviderCredentials {
+    baseUrl: string
+    apiKey: string
+}
+
+export function getModelProviderCredentials(headers: IncomingHttpHeaders): ModelProviderCredentials | undefined {
+  const baseUrl = getGPTScriptEnv(headers, 'OPENAI_BASE_URL')?.trim()
+  if (!baseUrl) return undefined
+
+  const apiKey = getGPTScriptEnv(headers, 'OPENAI_API_KEY')?.trim()
+  if (!apiKey) return undefined
+
+  return { baseUrl, apiKey }
+}
+
 export function getGPTScriptEnv (headers: IncomingHttpHeaders, envKey: string): string | undefined {
   const envHeader = headers?.['x-gptscript-env']
   const envArray = Array.isArray(envHeader) ? envHeader : [envHeader]

diff --git a/browser/tool.gpt b/browser/tool.gpt
@@ -1,9 +1,9 @@
 Name: Browser
 Description: Tools to navigate websites using a browser.
 Metadata: bundle: true
+Metadata: noUserAuth: sys.model.provider.credential
 Credentials: github.com/gptscript-ai/credentials/model-provider
 Share Tools: Browse, Get Page Contents, Filter, Fill, Enter, Scroll, Back, Forward, Screenshot, Download File From URL
-Metadata: noUserAuth: sys.model.provider.credential
 
 ---
 Name: Get Page Contents
@@ -43,6 +43,8 @@ Tools: service
 ---
 Name: Fill
 Metadata: index: false
+Metadata: noUserAuth: sys.model.provider.credential
+Credentials: github.com/gptscript-ai/credentials/model-provider
 Share Context: Browser Context
 Tools: service
 Description: Fills text into an element on the web page. Useful for filling out forms and other input fields.
@@ -181,6 +183,7 @@ END TOOL USAGE: The `screenshotInfo` response field
 
 ---
 Name: service
+Metadata: requestedEnvVars: OPENAI_API_KEY,OPENAI_BASE_URL
 Metadata: index: false
 
 #!sys.daemon /usr/bin/env npm --prefix ${GPTSCRIPT_TOOL_DIR} run server