Shopify · endoze · May 16, 2025
diff --git a/lib/roast.rb b/lib/roast.rb
@@ -3,6 +3,7 @@
 require "raix"
 require "thor"
 require "roast/version"
+require "roast/errors"
 require "roast/tools"
 require "roast/helpers"
 require "roast/resources"

diff --git a/lib/roast/errors.rb b/lib/roast/errors.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Roast
+  # Custom error for API resource not found (404) responses
+  class ResourceNotFoundError < StandardError; end
+
+  # Custom error for when API authentication fails
+  class AuthenticationError < StandardError; end
+end
diff --git a/lib/roast/workflow/base_workflow.rb b/lib/roast/workflow/base_workflow.rb
@@ -105,17 +105,15 @@ def chat_completion(**kwargs)
           })
           result
         end
+      rescue Faraday::ResourceNotFound => e
+        execution_time = Time.now - start_time
+        message = e.response.dig(:body, "error", "message") || e.message
+        error = Roast::ResourceNotFoundError.new(message)
+        e.set_backtrace([])
+        log_and_raise_error(error, message, step_model, kwargs, execution_time)
       rescue => e
         execution_time = Time.now - start_time
-
-        ActiveSupport::Notifications.instrument("roast.chat_completion.error", {
-          error: e.class.name,
-          message: e.message,
-          model: step_model,
-          parameters: kwargs.except(:openai, :model),
-          execution_time: execution_time,
-        })
-        raise
+        log_and_raise_error(e, e.message, step_model, kwargs, execution_time)
       end
 
       def workflow
@@ -124,6 +122,18 @@ def workflow
 
       private
 
+      def log_and_raise_error(error, message, model, params, execution_time)
+        ActiveSupport::Notifications.instrument("roast.chat_completion.error", {
+          error: error.class.name,
+          message: message,
+          model: model,
+          parameters: params.except(:openai, :model),
+          execution_time: execution_time,
+        })
+
+        raise error
+      end
+
       # Determine the directory where the actual class is defined, not BaseWorkflow
       def determine_context_path
         # Get the actual class's source file

diff --git a/lib/roast/workflow/configuration_parser.rb b/lib/roast/workflow/configuration_parser.rb
@@ -104,29 +104,37 @@ def load_roast_initializers
       end
 
       def configure_api_client
-        return unless configuration.api_token
+        client = case configuration.api_provider
+        when :openrouter
+          $stderr.puts "Configuring OpenRouter client with token from workflow"
+          require "open_router"
 
-        begin
-          case configuration.api_provider
-          when :openrouter
-            $stderr.puts "Configuring OpenRouter client with token from workflow"
-            require "open_router"
-
-            Raix.configure do |config|
-              config.openrouter_client = OpenRouter::Client.new(access_token: configuration.api_token)
-            end
-          else
-            $stderr.puts "Configuring OpenAI client with token from workflow"
-            require "openai"
+          Raix.configure do |config|
+            config.openrouter_client = OpenRouter::Client.new(access_token: configuration.api_token)
+          end
+        else
+          $stderr.puts "Configuring OpenAI client with token from workflow"
+          require "openai"
 
-            Raix.configure do |config|
-              config.openai_client = OpenAI::Client.new(access_token: configuration.api_token)
-            end
+          Raix.configure do |config|
+            config.openai_client = OpenAI::Client.new(access_token: configuration.api_token)
           end
-        rescue => e
-          Roast::Helpers::Logger.error("Error configuring API client: #{e.message}")
-          # Don't fail the workflow if client can't be configured
         end
+
+        # use an api call to list models as a way to validate our client configuration
+        client.models.list
+      rescue OpenRouter::ConfigurationError, Faraday::UnauthorizedError => e
+        error = Roast::AuthenticationError.new("API authentication failed: No API token provided or token is invalid")
+        e.set_backtrace([])
+
+        ActiveSupport::Notifications.instrument("roast.workflow.start.error", {
+          error: error.class.name,
+          message: error.message,
+        })
+
+        raise error
+      rescue => e
+        Roast::Helpers::Logger.error("Error configuring API client: #{e.message}")
       end
 
       def load_state_and_update_steps(steps, skip_until, step_name, timestamp)

diff --git a/test/fixtures/files/workflow/workflow.yml b/test/fixtures/files/workflow/workflow.yml
@@ -1,4 +1,6 @@
 name: Grading current test changes
+api_provider: openai
+api_token: test_openai_token
 
 tools:
   - Roast::Tools::Grep
@@ -19,4 +21,4 @@ verify_test_helpers:
 
 verify_mocks_and_stubs:
   json: true
-  print_response: true
+  print_response: true
diff --git a/test/fixtures/targetless_workflow.yml b/test/fixtures/targetless_workflow.yml
@@ -1,4 +1,7 @@
 name: Targetless Workflow
+api_provider: openai
+api_token: test_openai_token
+
 tools: 
   - Roast::Tools::Grep
   - Roast::Tools::ReadFile

diff --git a/test/roast/helpers/prompt_loader_test.rb b/test/roast/helpers/prompt_loader_test.rb
@@ -7,6 +7,10 @@ class RoastHelpersPromptLoaderTest < ActiveSupport::TestCase
   def setup
     @workflow_file = fixture_file("workflow/workflow.yml")
     @test_file = fixture_file("test.rb")
+    mock_openai_client = mock
+    OpenAI::Client.stubs(:new).with(access_token: "test_openai_token").returns(mock_openai_client)
+    mock_openai_client.stubs(:models).returns(mock_openai_client)
+    mock_openai_client.stubs(:list).returns([])
     @workflow = build_workflow(@workflow_file, @test_file)
   end
 

diff --git a/test/roast/workflow/base_workflow_test.rb b/test/roast/workflow/base_workflow_test.rb
@@ -8,14 +8,28 @@ class RoastWorkflowBaseWorkflowTest < ActiveSupport::TestCase
   FILE_PATH = File.join(Dir.pwd, "test/fixtures/files/test.rb")
 
   def setup
-    # Use Mocha for stubbing/mocking
     Roast::Helpers::PromptLoader.stubs(:load_prompt).returns("Test prompt")
     Roast::Tools.stubs(:setup_interrupt_handler)
+    Roast::Tools.stubs(:setup_exit_handler)
+    ActiveSupport::Notifications.stubs(:instrument).returns(true)
+
+    @mock_client = mock("client")
+    mock_config = mock("config")
+    mock_config.stubs(:openrouter_client).returns(@mock_client)
+    mock_config.stubs(:openai_client).returns(@mock_client)
+    mock_config.stubs(:model).returns("gpt-4")
+    mock_config.stubs(:max_tokens).returns(1024)
+    mock_config.stubs(:max_completion_tokens).returns(1024)
+    mock_config.stubs(:temperature).returns(0.7)
+    Raix.stubs(:configuration).returns(mock_config)
   end
 
   def teardown
     Roast::Helpers::PromptLoader.unstub(:load_prompt)
     Roast::Tools.unstub(:setup_interrupt_handler)
+    Roast::Tools.unstub(:setup_exit_handler)
+    ActiveSupport::Notifications.unstub(:instrument)
+    Raix.unstub(:configuration)
   end
 
   test "initializes with file and sets up transcript" do
@@ -39,4 +53,44 @@ def teardown
     workflow.append_to_final_output("Test output")
     assert_equal "Test output", workflow.final_output
   end
+
+  test "handles ResourceNotFoundError correctly when Faraday::ResourceNotFound is raised" do
+    workflow = Roast::Workflow::BaseWorkflow.new(FILE_PATH)
+
+    mock_response = { body: { "error" => { "message" => "Model not found" } } }
+    faraday_error = Faraday::ResourceNotFound.new(nil)
+    faraday_error.stubs(:response).returns(mock_response)
+
+    @mock_client.expects(:complete).raises(faraday_error)
+
+    ActiveSupport::Notifications.expects(:instrument).with("roast.chat_completion.start", anything).once
+    ActiveSupport::Notifications.expects(:instrument).with(
+      "roast.chat_completion.error",
+      has_entry(error: "Roast::ResourceNotFoundError"),
+    ).once
+
+    assert_raises(Roast::ResourceNotFoundError) do
+      workflow.chat_completion
+    end
+  end
+
+  test "handles other errors properly without conversion" do
+    workflow = Roast::Workflow::BaseWorkflow.new(FILE_PATH)
+
+    standard_error = StandardError.new("Some other error")
+
+    @mock_client.expects(:complete).raises(standard_error)
+
+    ActiveSupport::Notifications.expects(:instrument).with("roast.chat_completion.start", anything).once
+    ActiveSupport::Notifications.expects(:instrument).with(
+      "roast.chat_completion.error",
+      has_entry(error: "StandardError"),
+    ).once
+
+    error = assert_raises(StandardError) do
+      workflow.chat_completion
+    end
+
+    assert_equal "Some other error", error.message
+  end
 end
diff --git a/test/roast/workflow/configuration_parser_openrouter_test.rb b/test/roast/workflow/configuration_parser_openrouter_test.rb
@@ -8,14 +8,15 @@ module Workflow
     class ConfigurationParserOpenRouterTest < Minitest::Test
       def setup
         @workflow_path = File.expand_path("../../fixtures/files/openrouter_workflow.yml", __dir__)
+        mock_openrouter_client = mock
+        OpenRouter::Client.stubs(:new).with(access_token: "test_openrouter_token").returns(mock_openrouter_client)
+        mock_openrouter_client.stubs(:models).returns(mock_openrouter_client)
+        mock_openrouter_client.stubs(:list).returns([])
       end
 
       def test_configure_openrouter_client
         setup_openrouter_constants
 
-        mock_openrouter_client = mock
-        OpenRouter::Client.stubs(:new).with(access_token: "test_openrouter_token").returns(mock_openrouter_client)
-
         ConfigurationParser.new(@workflow_path)
       end
 
@@ -30,7 +31,7 @@ def setup_openrouter_constants
       end
 
       def teardown
-        OpenRouter.send(:remove_const, :Client) if defined?(::OpenRouter) && defined?(::OpenRouter::Client)
+        OpenRouter::Client.unstub(:new) if OpenRouter::Client.respond_to?(:unstub)
       end
     end
   end

@@ -8,9 +8,18 @@
 class RoastWorkflowConfigurationParserTest < ActiveSupport::TestCase
   def setup
     @workflow_path = fixture_file("workflow/workflow.yml")
+    mock_openai_client = mock
+    OpenAI::Client.stubs(:new).with(access_token: "test_openai_token").returns(mock_openai_client)
+    mock_openai_client.stubs(:models).returns(mock_openai_client)
+    mock_openai_client.stubs(:list).returns([])
     @parser = Roast::Workflow::ConfigurationParser.new(@workflow_path)
   end
 
+  def teardown
+    OpenAI::Client.unstub(:new) if OpenAI::Client.respond_to?(:unstub)
+    OpenRouter::Client.unstub(:new) if OpenRouter::Client.respond_to?(:unstub)
+  end
+
   def capture_stderr
     old_stderr = $stderr
     $stderr = StringIO.new
@@ -25,6 +34,16 @@ def test_initialize_with_example_workflow
     assert_equal("run_coverage", @parser.configuration.steps.first)
   end
 
+  def test_no_api_key_provided
+    workflow_path = fixture_file("openrouter_no_api_token_workflow.yml")
+
+    error = assert_raises(Roast::AuthenticationError) do
+      Roast::Workflow::ConfigurationParser.new(workflow_path)
+    end
+
+    assert_equal("API authentication failed: No API token provided or token is invalid", error.message)
+  end
+
   def test_begin_without_files_or_target_runs_targetless_workflow
     executor = mock("WorkflowExecutor")
     executor.stubs(:execute_steps)

@@ -10,6 +10,10 @@
 class RoastWorkflowTargetlessWorkflowTest < ActiveSupport::TestCase
   def setup
     @workflow_path = fixture_file_path("targetless_workflow.yml")
+    mock_openai_client = mock
+    OpenAI::Client.stubs(:new).with(access_token: "test_openai_token").returns(mock_openai_client)
+    mock_openai_client.stubs(:models).returns(mock_openai_client)
+    mock_openai_client.stubs(:list).returns([])
     @parser = Roast::Workflow::ConfigurationParser.new(@workflow_path)
   end