From d8d72e46fc18d4570dfa20082fb24640f830fa38 Mon Sep 17 00:00:00 2001 From: YBubu Date: Wed, 14 May 2025 10:52:29 +0000 Subject: [PATCH 1/5] Proxy playground agent (still WIP) --- api/api/routers/agents/meta_agent.py | 52 ++ api/api/routers/integrations_router.py | 4 +- .../internal_tasks/integration_service.py | 4 +- .../integration_service_test.py | 12 +- .../internal_tasks/meta_agent_service.py | 461 ++++++++++++++ .../agents/input_variables_extractor_agent.py | 36 ++ api/core/agents/integration_agent.py | 2 +- api/core/agents/meta_agent_proxy.py | 600 ++++++++++++++++++ .../agents/output_schema_extractor_agent.py | 41 ++ api/core/domain/integration_domain.py | 109 ---- .../instruction_python_snippets.py | 52 ++ .../integration_domain/integration_domain.py | 121 ++++ .../openai_sdk_python_snippets.py | 73 +++ .../openai_sdk_ts_snippets.py | 92 +++ api/core/domain/integration_domain_test.py | 19 +- .../storage/clickhouse/clickhouse_client.py | 22 +- .../mongo/partials/mongo_agent_runs.py | 7 +- api/core/storage/task_run_storage.py | 7 +- docs/developers/js/openai.md | 407 ++++++++++++ docs/developers/python/openai.md | 273 ++++++++ 20 files changed, 2258 insertions(+), 136 deletions(-) create mode 100644 api/core/agents/input_variables_extractor_agent.py create mode 100644 api/core/agents/meta_agent_proxy.py create mode 100644 api/core/agents/output_schema_extractor_agent.py delete mode 100644 api/core/domain/integration_domain.py create mode 100644 api/core/domain/integration_domain/instruction_python_snippets.py create mode 100644 api/core/domain/integration_domain/integration_domain.py create mode 100644 api/core/domain/integration_domain/openai_sdk_python_snippets.py create mode 100644 api/core/domain/integration_domain/openai_sdk_ts_snippets.py create mode 100644 docs/developers/js/openai.md create mode 100644 docs/developers/python/openai.md diff --git a/api/api/routers/agents/meta_agent.py b/api/api/routers/agents/meta_agent.py index ae31dddaa..86da7aad2 100644 --- a/api/api/routers/agents/meta_agent.py +++ b/api/api/routers/agents/meta_agent.py @@ -88,6 +88,7 @@ async def get_meta_agent_chat( user_properties: UserPropertiesDep, meta_agent_service: MetaAgentServiceDep, ) -> StreamingResponse: + """ async def _stream() -> AsyncIterator[BaseModel]: async for messages in meta_agent_service.stream_meta_agent_response( task_tuple=task_tuple, @@ -97,5 +98,56 @@ async def _stream() -> AsyncIterator[BaseModel]: playground_state=request.playground_state, ): yield MetaAgentChatResponse(messages=messages) + """ + + # TODO: plug back the legacy proxy meta agent + async def _proxy_stream() -> AsyncIterator[BaseModel]: + async for messages in meta_agent_service.stream_proxy_meta_agent_response( + task_tuple=task_tuple, + agent_schema_id=request.schema_id, + user_email=user_properties.user_email, + messages=request.messages, + ): + yield MetaAgentChatResponse(messages=messages) + + return safe_streaming_response(_proxy_stream) + + +class ProxyMetaAgentChatRequest(BaseModel): + # Schema id is passed here instead of as a path parameters in order to have the endpoint schema-agnostic since + # the schema id might change in the middle of the conversation based on the agent's actions. + schema_id: TaskSchemaID + messages: list[MetaAgentChatMessage] = Field( + description="The list of messages in the conversation, the last message being the most recent one", + ) + + +@router.post( + "/proxy/messages", + description="To chat with WorkflowAI's meta agent", + responses={ + 200: { + "content": { + "text/event-stream": { + "schema": MetaAgentChatResponse.model_json_schema(), + }, + }, + }, + }, +) +async def get_proxy_chat_answer( + task_tuple: TaskTupleDep, + request: ProxyMetaAgentChatRequest, + user_properties: UserPropertiesDep, + meta_agent_service: MetaAgentServiceDep, +) -> StreamingResponse: + async def _stream() -> AsyncIterator[BaseModel]: + async for messages in meta_agent_service.stream_proxy_meta_agent_response( + task_tuple=task_tuple, + agent_schema_id=request.schema_id, + user_email=user_properties.user_email, + messages=request.messages, + ): + yield MetaAgentChatResponse(messages=messages) return safe_streaming_response(_stream) diff --git a/api/api/routers/integrations_router.py b/api/api/routers/integrations_router.py index 3cf106a0f..6c55a90c2 100644 --- a/api/api/routers/integrations_router.py +++ b/api/api/routers/integrations_router.py @@ -10,8 +10,8 @@ IntegrationChatMessage, IntegrationChatResponse, ) -from core.domain.integration_domain import OFFICIAL_INTEGRATIONS, IntegrationKind -from core.domain.integration_domain import Integration as DomainIntegration +from core.domain.integration_domain.integration_domain import OFFICIAL_INTEGRATIONS, IntegrationKind +from core.domain.integration_domain.integration_domain import Integration as DomainIntegration from core.utils.stream_response_utils import safe_streaming_response router = APIRouter(prefix="/v1/integrations") diff --git a/api/api/services/internal_tasks/integration_service.py b/api/api/services/internal_tasks/integration_service.py index 3442939ab..eb13b7c66 100644 --- a/api/api/services/internal_tasks/integration_service.py +++ b/api/api/services/internal_tasks/integration_service.py @@ -27,7 +27,7 @@ from core.domain.errors import ObjectNotFoundError from core.domain.events import EventRouter from core.domain.fields.chat_message import ChatMessage -from core.domain.integration_domain import ( +from core.domain.integration_domain.integration_domain import ( OFFICIAL_INTEGRATIONS, PROPOSED_AGENT_NAME_AND_MODEL_PLACEHOLDER, WORKFLOWAI_API_KEY_PLACEHOLDER, @@ -255,7 +255,7 @@ async def _find_relevant_run_and_agent( The goals of this functions is to spot a run / agent that was (very likely) created by the user currently doing the onboarding flow. """ - async for run in self.storage.task_runs.list_runs_since( + async for run in self.storage.task_runs.list_latest_runs( since_date=discussion_started_at, is_active=True, # TODO: filter on proxy runs limit=2, # TODO: pick a better limit diff --git a/api/api/services/internal_tasks/integration_service_test.py b/api/api/services/internal_tasks/integration_service_test.py index 2d1bd65c2..766c086c3 100644 --- a/api/api/services/internal_tasks/integration_service_test.py +++ b/api/api/services/internal_tasks/integration_service_test.py @@ -23,7 +23,7 @@ from core.domain.agent_run import AgentRun from core.domain.errors import ObjectNotFoundError from core.domain.events import EventRouter -from core.domain.integration_domain import ( +from core.domain.integration_domain.integration_domain import ( OFFICIAL_INTEGRATIONS, Integration, IntegrationKind, @@ -285,8 +285,8 @@ async def test_find_relevant_run_no_runs( integration_service: IntegrationService, ): start_time = datetime.datetime.now() - # Mock list_runs_since to return an async iterator - integration_service.storage.task_runs.list_runs_since = lambda *args, **kwargs: mock_aiter() # type: ignore[reportUnknownLambdaType] + # Mock list_latest_runs to return an async iterator + integration_service.storage.task_runs.list_latest_runs = lambda *args, **kwargs: mock_aiter() # type: ignore[reportUnknownLambdaType] result = await integration_service._find_relevant_run_and_agent(start_time) # pyright: ignore[reportPrivateUsage] assert result is None @@ -306,7 +306,7 @@ async def test_find_relevant_run_default_agent( mock_agent.name = DEFAULT_AGENT_ID # Setup the storage mock to return our mock run and agent - integration_service.storage.task_runs.list_runs_since = lambda *args, **kwargs: mock_aiter(mock_run) # type: ignore[reportUnknownLambdaType] + integration_service.storage.task_runs.list_latest_runs = lambda *args, **kwargs: mock_aiter(mock_run) # type: ignore[reportUnknownLambdaType] # Patch the _get_agent_by_uid method to return our mock agent with patch.object( @@ -337,7 +337,7 @@ async def test_find_relevant_run_new_named_agent( mock_agent.created_at = start_time + datetime.timedelta(minutes=5) # Setup the storage mock to return our mock run and agent - integration_service.storage.task_runs.list_runs_since = lambda *args, **kwargs: mock_aiter(mock_run) # type: ignore[reportUnknownLambdaType] + integration_service.storage.task_runs.list_latest_runs = lambda *args, **kwargs: mock_aiter(mock_run) # type: ignore[reportUnknownLambdaType] # Patch the _get_agent_by_uid method to return our mock agent with patch.object( @@ -368,7 +368,7 @@ async def test_find_relevant_run_ignore_old_named_agent( mock_agent.created_at = start_time - datetime.timedelta(minutes=5) # Setup the storage mock to return our mock run and agent - integration_service.storage.task_runs.list_runs_since = lambda *args, **kwargs: mock_aiter(mock_run) # pyright: ignore[reportAttributeAccessIssue, reportUnknownLambdaType] + integration_service.storage.task_runs.list_latest_runs = lambda *args, **kwargs: mock_aiter(mock_run) # pyright: ignore[reportAttributeAccessIssue, reportUnknownLambdaType] # Patch the _get_agent_by_uid method to return our mock agent with patch.object( diff --git a/api/api/services/internal_tasks/meta_agent_service.py b/api/api/services/internal_tasks/meta_agent_service.py index f9c2a3de0..7bf98e9e9 100644 --- a/api/api/services/internal_tasks/meta_agent_service.py +++ b/api/api/services/internal_tasks/meta_agent_service.py @@ -19,6 +19,7 @@ ExtractCompanyInfoFromDomainTaskOutput, safe_generate_company_description_from_email, ) +from core.agents.input_variables_extractor_agent import InputVariablesExtractorInput, input_variables_extractor_agent from core.agents.meta_agent import ( META_AGENT_INSTRUCTIONS, EditSchemaToolCallResult, @@ -36,14 +37,35 @@ from core.agents.meta_agent import ( PlaygroundState as PlaygroundStateDomain, ) +from core.agents.meta_agent_proxy import ( + GENERIC_PROPOSED_INSTRUCTIONS_INSTRUCTIONS, + PROPOSE_INPUT_VARIABLES_INSTRUCTIONS, + PROPOSE_NON_OPENAI_MODELS_INSTRUCTIONS, + PROPOSE_STRUCTURED_OUTPUT_INSTRUCTIONS, + ProxyMetaAgentInput, + proxy_meta_agent_proxy, +) +from core.agents.meta_agent_proxy import PlaygroundState as ProxyPlaygroundStateDomain +from core.agents.meta_agent_proxy import ( + ProxyMetaAgentChatMessage as ProxyMetaAgentChatMessageDomain, +) from core.agents.meta_agent_user_confirmation_agent import ( MetaAgentUserConfirmationInput, meta_agent_user_confirmation_agent, ) +from core.agents.output_schema_extractor_agent import OutputSchemaExtractorInput, output_schema_extractor_agent from core.domain.agent_run import AgentRun from core.domain.events import EventRouter, MetaAgentChatMessagesSent from core.domain.fields.chat_message import ChatMessage from core.domain.fields.file import File +from core.domain.integration_domain.integration_domain import ( + ProgrammingLanguage, + default_integration_for_language, +) +from core.domain.models.model_data import LatestModel +from core.domain.models.model_datas_mapping import MODEL_DATAS +from core.domain.models.model_provider_datas_mapping import AZURE_PROVIDER_DATA, OPENAI_PROVIDER_DATA +from core.domain.models.models import Model from core.domain.page import Page from core.domain.task_variant import SerializableTaskVariant from core.domain.url_content import URLContent @@ -77,6 +99,24 @@ class MetaAgentContext(NamedTuple): reviewed_input_count: int | None +class ProxyMetaAgentContext(NamedTuple): + company_description: ExtractCompanyInfoFromDomainTaskOutput | None + existing_agents: list[str] | None + agent_runs: list[AgentRun] | None + feedback_page: Page[ProxyMetaAgentInput.AgentLifecycleInfo.FeedbackInfo.AgentFeedback] | None + has_active_runs: HasActiveRunAndDate | None + reviewed_input_count: int | None + + +def get_programming_language_for_user_agent(user_agent: str) -> ProgrammingLanguage: + user_agent = user_agent.lower() + if "python" in user_agent: + return ProgrammingLanguage.PYTHON + if any(kw in user_agent for kw in ("javascript", "js", "typescript", "ts")): + return ProgrammingLanguage.TYPESCRIPT + return ProgrammingLanguage.PYTHON + + class MetaAgentToolCall(BaseModel): tool_name: str = "" @@ -231,6 +271,14 @@ def to_domain(self) -> GenerateAgentInputToolCallResult: ) +MetaAgentChatMessageKind: TypeAlias = Literal[ + "non_specific", + "try_other_models_proposal", + "setup_input_variables_proposal", + "setup_structured_output_proposal", +] + + class MetaAgentChatMessage(BaseModel): role: Literal["USER", "PLAYGROUND", "ASSISTANT"] = Field( description="The role of the message sender, 'USER' is the actual human user browsing the playground, 'PLAYGROUND' are automated messages sent by the playground to the agent, and 'ASSISTANT' being the assistant generated by the agent", @@ -250,6 +298,11 @@ class MetaAgentChatMessage(BaseModel): feedback_token: str | None = None + kind: MetaAgentChatMessageKind = Field( + default="non_specific", + description="The kind of message, used to determine the kind of tool call to show in the frontend.", + ) + def to_domain(self) -> MetaAgentChatMessageDomain: return MetaAgentChatMessageDomain( role=self.role, @@ -258,6 +311,12 @@ def to_domain(self) -> MetaAgentChatMessageDomain: tool_call_status=self.tool_call.status if self.tool_call else None, ) + def to_proxy_domain(self) -> ProxyMetaAgentChatMessageDomain: + return ProxyMetaAgentChatMessageDomain( + role=self.role, + content=self.content, + ) + def to_chat_message(self) -> ChatMessage: role_map: dict[Literal["USER", "PLAYGROUND", "ASSISTANT"], Literal["USER", "ASSISTANT"]] = { "USER": "USER", @@ -866,3 +925,405 @@ async def stream_meta_agent_response( yield ret self.dispatch_new_assistant_messages_event(ret) + + # TODO: delete when we'll have factorized the two agents + async def _proxy_build_model_list( + self, + instructions: str | None, + current_agent: SerializableTaskVariant, + ) -> list[ProxyPlaygroundStateDomain.PlaygroundModel]: + models = await self.models_service.models_for_task( + current_agent, + instructions=instructions, + requires_tools=None, + ) + return [ + ProxyPlaygroundStateDomain.PlaygroundModel( + id=model.id, + name=model.name, + quality_index=model.quality_index, + context_window_tokens=model.context_window_tokens, + is_not_supported_reason=model.is_not_supported_reason or "", + estimate_cost_per_thousand_runs_usd=round(model.average_cost_per_run_usd * 1000, 3) + if model.average_cost_per_run_usd + else None, + is_default=model.is_default, + is_latest=model.is_latest, + ) + for model in models + ] + + # TODO: delete when we'll have factorized the two agents + async def list_proxy_deployments( + self, + task_tuple: TaskTuple, + agent_schema_id: int, + ) -> list[ProxyMetaAgentInput.AgentLifecycleInfo.DeploymentInfo.Deployment]: + versions = await self.versions_service.list_version_majors(task_tuple, agent_schema_id, self.models_service) + + deployments: list[ProxyMetaAgentInput.AgentLifecycleInfo.DeploymentInfo.Deployment] = [] + + for version in versions: + for minor in version.minors or []: + deployments.extend( + [ + ProxyMetaAgentInput.AgentLifecycleInfo.DeploymentInfo.Deployment( + environment=deployment.environment, + deployed_at=deployment.deployed_at, + deployed_by_email=deployment.deployed_by.user_email if deployment.deployed_by else None, + model_used=minor.properties.model, + last_active_at=minor.last_active_at, + run_count=minor.run_count, + notes=minor.notes, + ) + for deployment in minor.deployments or [] + ], + ) + + return deployments + + async def _fetch_proxy_meta_agent_context( + self, + task_tuple: TaskTuple, + agent_schema_id: int, + user_email: str | None, + ) -> ProxyMetaAgentContext: + """ + Fetch all context data needed for the meta agent input, handling exceptions. + + If any individual fetch fails, it returns None for that part of the context + instead of failing the entire operation. + """ + context_results = await asyncio.gather( + safe_generate_company_description_from_email(user_email), + list_agent_summaries(self.storage, limit=10), + self._fetch_latest_agent_runs(task_tuple), + self.feedback_service.list_feedback( + task_tuple[1], + run_id=None, + limit=10, + offset=0, + map_fn=ProxyMetaAgentInput.AgentLifecycleInfo.FeedbackInfo.AgentFeedback.from_domain, + ), + self.has_active_agent_runs(task_tuple, agent_schema_id), + self.get_reviewed_input_count(task_tuple, agent_schema_id), + return_exceptions=True, + ) + + # Process each result - for each item, either use the value or log and return a default value + if isinstance(context_results[0], BaseException): + self._logger.warning("Failed to fetch company_description", exc_info=context_results[0]) + company_description = None + else: + company_description = context_results[0] + + if isinstance(context_results[1], BaseException): + self._logger.warning("Failed to fetch existing_agents", exc_info=context_results[1]) + existing_agents = None + else: + existing_agents = [str(agent) for agent in context_results[1] or []] + + if isinstance(context_results[2], BaseException): + self._logger.warning("Failed to fetch agent_runs", exc_info=context_results[2]) + agent_runs = None + else: + agent_runs = context_results[2] + + if isinstance(context_results[3], BaseException): + self._logger.warning("Failed to fetch feedback_page", exc_info=context_results[3]) + feedback_page = None + else: + feedback_page = context_results[3] + + if isinstance(context_results[4], BaseException): + self._logger.warning("Failed to fetch has_active_runs", exc_info=context_results[4]) + has_active_runs = None + else: + has_active_runs = context_results[4] + + if isinstance(context_results[5], BaseException): + self._logger.warning("Failed to fetch reviewed_input_count", exc_info=context_results[5]) + reviewed_input_count = None + else: + reviewed_input_count = context_results[5] + + return ProxyMetaAgentContext( + company_description=company_description, + existing_agents=existing_agents, + agent_runs=agent_runs, + feedback_page=feedback_page, + has_active_runs=has_active_runs, + reviewed_input_count=reviewed_input_count, + ) + + async def _build_proxy_meta_agent_input( + self, + task_tuple: TaskTuple, + agent_schema_id: int, + user_email: str | None, + messages: list[MetaAgentChatMessage], + current_agent: SerializableTaskVariant, + ) -> tuple[ProxyMetaAgentInput, list[AgentRun]]: + # Fetch context data with exception handling + context = await self._fetch_proxy_meta_agent_context( + task_tuple, + agent_schema_id, + user_email, + ) + + """ + # Extract files from agent_input + agent_input_schema = current_agent.input_schema.json_schema.copy() + agent_input_copy, agent_input_files = self._extract_files_from_agent_input( + playground_state.agent_input, + agent_input_schema, + ) + """ + return ProxyMetaAgentInput( + current_datetime=datetime.datetime.now(), + messages=[message.to_proxy_domain() for message in messages], + latest_messages_url_content=await self._extract_url_content_from_messages(messages), + company_context=ProxyMetaAgentInput.CompanyContext( + company_name=context.company_description.company_name if context.company_description else None, + company_description=context.company_description.description if context.company_description else None, + company_locations=context.company_description.locations if context.company_description else None, + company_industries=context.company_description.industries if context.company_description else None, + company_products=context.company_description.products if context.company_description else None, + existing_agents_descriptions=context.existing_agents or [], + ), + workflowai_documentation_sections=await DocumentationService().get_relevant_doc_sections( + chat_messages=[message.to_chat_message() for message in messages], + agent_instructions=GENERIC_PROPOSED_INSTRUCTIONS_INSTRUCTIONS or "", + ), + integration_documentation=[], # Will be filled in later + available_tools_description=internal_tools_description( + include={ToolKind.WEB_BROWSER_TEXT, ToolKind.WEB_SEARCH_PERPLEXITY_SONAR_PRO}, + ), + playground_state=ProxyPlaygroundStateDomain( + current_agent=ProxyPlaygroundStateDomain.Agent( + name=current_agent.name, + slug=current_agent.task_id, + schema_id=current_agent.task_schema_id, + description=current_agent.description, + input_schema=current_agent.input_schema.json_schema, + output_schema=current_agent.output_schema.json_schema, + ), + available_models=await self._proxy_build_model_list("", current_agent), # TODO: add instructions + agent_runs=[ + ProxyPlaygroundStateDomain.AgentRun( + id=agent_run.id, + model=agent_run.group.properties.model or "", + output=str(agent_run.task_output), # Handle both dict output and str + error=agent_run.error.model_dump() if agent_run.error else None, + cost_usd=agent_run.cost_usd, + duration_seconds=agent_run.duration_seconds, + user_evaluation=agent_run.user_review, + tool_calls=[ + ProxyPlaygroundStateDomain.AgentRun.ToolCall( + name=tool_call.tool_name, + input=tool_call.tool_input_dict, + ) + for llm_completion in agent_run.llm_completions or [] + for tool_call in llm_completion.tool_calls or [] + ], + raw_run_request=agent_run.metadata.get( + "initial_request_body", # TODO: put "initial_request_body" in a variable + None, + ) + if agent_run.metadata + else None, + ) + for agent_run in context.agent_runs + ] + if context.agent_runs + else None, + ), + agent_lifecycle_info=ProxyMetaAgentInput.AgentLifecycleInfo( + deployment_info=ProxyMetaAgentInput.AgentLifecycleInfo.DeploymentInfo( + has_api_or_sdk_runs=context.has_active_runs.has_active_runs, + latest_api_or_sdk_run_date=context.has_active_runs.latest_active_run_date, + deployments=await self.list_proxy_deployments(task_tuple, agent_schema_id), + ) + if context.has_active_runs + else None, + feedback_info=ProxyMetaAgentInput.AgentLifecycleInfo.FeedbackInfo( + user_feedback_count=context.feedback_page.count, + latest_user_feedbacks=context.feedback_page.items, + ) + if context.feedback_page + else None, + internal_review_info=ProxyMetaAgentInput.AgentLifecycleInfo.InternalReviewInfo( + reviewed_input_count=context.reviewed_input_count, + ), + ), + ), context.agent_runs or [] + + async def _fetch_latest_agent_runs( + self, + task_tuple: TaskTuple, + ) -> list[AgentRun]: + return [ + run + async for run in self.storage.task_runs.list_latest_runs( + task_uid=task_tuple[1], + limit=10, + ) + ] + + def _is_user_triggered(self, messages: list[MetaAgentChatMessage]) -> bool: + if len(messages) == 0: + return False + + latest_message = messages[-1] + + return latest_message.role == "USER" + + def _is_only_using_openai_models(self, agent_runs: list[AgentRun]) -> bool: + for agent_run in agent_runs: + if not agent_run.group.properties.model: + continue + try: + model = Model(agent_run.group.properties.model) + except ValueError: + self._logger.warning("Failed to parse model", extra={"model": agent_run.group.properties.model}) + continue + + model_data = MODEL_DATAS[model] + if isinstance(model_data, LatestModel): + model = model_data.model + model_data = MODEL_DATAS[model_data.model] + + if model not in AZURE_PROVIDER_DATA.keys() and model not in OPENAI_PROVIDER_DATA.keys(): + return False + + return True + + async def stream_proxy_meta_agent_response( + self, + task_tuple: TaskTuple, + agent_schema_id: int, + user_email: str | None, + messages: list[MetaAgentChatMessage], + ) -> AsyncIterator[list[MetaAgentChatMessage]]: + if len(messages) == 0: + yield [MetaAgentChatMessage(role="ASSISTANT", content=FIRST_MESSAGE_CONTENT)] + return + + current_agent = await self.storage.task_variant_latest_by_schema_id(task_tuple[0], agent_schema_id) + + self.dispatch_new_user_messages_event(messages) + + proxy_meta_agent_input, agent_runs = await self._build_proxy_meta_agent_input( + task_tuple, + agent_schema_id, + user_email, + messages, + current_agent, + ) + + # TODO: reuse the integration used in the onboarding (ex: Instructor Python) + user_agent = "" + if len(agent_runs) and agent_runs[0].metadata: + user_agent = agent_runs[0].metadata.get("user-agent", "") + + programming_language = get_programming_language_for_user_agent(user_agent) + integration = default_integration_for_language(programming_language) + + # Fill the agent input with the right documentations + proxy_meta_agent_input.integration_documentation = DocumentationService().get_documentation_by_path( + integration.documentation_filepaths, + ) + proxy_meta_agent_input.playground_state.current_agent.used_integration = integration + + # TODO: remove messages[-1].content not in ["POLL", "poll"] + is_user_triggered = self._is_user_triggered(messages) and messages[-1].content not in ["POLL", "poll"] + is_using_instruction_variables = current_agent.input_schema.json_schema.get("properties", None) is not None + is_using_structured_generation = not current_agent.output_schema.json_schema.get("format") == "message" + + if not is_user_triggered and agent_runs and self._is_only_using_openai_models(agent_runs): + # "Try non-OpenAI model" use case + + instructions = PROPOSE_NON_OPENAI_MODELS_INSTRUCTIONS + elif not is_user_triggered and agent_runs and not is_using_instruction_variables: + # "Migrate to input variables" use case + input_variables_extractor_agent_run = await input_variables_extractor_agent( + InputVariablesExtractorInput( + agent_inputs=[agent_run.task_input for agent_run in agent_runs], + ), + ) + proxy_meta_agent_input.suggested_instructions_with_input_variables = ( + input_variables_extractor_agent_run.instructions_with_input_variables + ) + proxy_meta_agent_input.suggested_input_variables_example = ( + input_variables_extractor_agent_run.input_variables_example + ) + instructions = PROPOSE_INPUT_VARIABLES_INSTRUCTIONS + elif not is_user_triggered and agent_runs and not is_using_structured_generation: + # "Migrate to structured output" use case + + output_schema_extractor_run = await output_schema_extractor_agent( + OutputSchemaExtractorInput( + agent_runs=[ + OutputSchemaExtractorInput.AgentRun( + raw_messages=[ + llm_completion.messages or [] for llm_completion in agent_run.llm_completions or [] + ], + input=str(agent_run.task_input), + output=str(agent_run.task_output), + ) + for agent_run in agent_runs + ], + programming_language=integration.programming_language, + structured_object_class=integration.output_class, + ), + ) + proxy_meta_agent_input.suggested_output_class_code = ( + output_schema_extractor_run.proposed_structured_object_class + ) + proxy_meta_agent_input.suggested_instructions_parts_to_remove = ( + output_schema_extractor_run.instructions_parts_to_remove + ) + instructions = PROPOSE_STRUCTURED_OUTPUT_INSTRUCTIONS + # All other use cases + else: + instructions = GENERIC_PROPOSED_INSTRUCTIONS_INSTRUCTIONS + + ret: list[MetaAgentChatMessage] = [] + + """ + chunk: "workflowai.Run[ProxyMetaAgentOutput] | None" = None + async for chunk in proxy_meta_agent.stream( + proxy_meta_agent_input, + version=workflowai.VersionProperties( + model=model, + instructions=instructions, + temperature=0.5, + ), + ): + if chunk.output.assistant_answer: + ret = [ + MetaAgentChatMessage( + role="ASSISTANT", + content=chunk.output.assistant_answer.replace("REPLACE_INSTRUCTIONS", instructions_to_inject), + feedback_token=chunk.feedback_token, + ), + ] + yield ret + """ + + accumulator = "" + async for chunk in proxy_meta_agent_proxy( + proxy_meta_agent_input, + instructions, + ): + if chunk.assistant_answer: + accumulator += chunk.assistant_answer + ret = [ + MetaAgentChatMessage( + role="ASSISTANT", + content=accumulator, + ), + ] + yield ret + + self.dispatch_new_assistant_messages_event(ret) diff --git a/api/core/agents/input_variables_extractor_agent.py b/api/core/agents/input_variables_extractor_agent.py new file mode 100644 index 000000000..3a15e2255 --- /dev/null +++ b/api/core/agents/input_variables_extractor_agent.py @@ -0,0 +1,36 @@ +from typing import Any + +import workflowai +from pydantic import BaseModel + + +class InputVariablesExtractorInput(BaseModel): + agent_inputs: list[dict[str, Any]] + + +class InputVariablesExtractorOutput(BaseModel): + instructions_with_input_variables: str + input_variables_example: dict[str, Any] + + +INSTRUCTIONS = """ +You are an expert at extracting input variables from agent instructions contained in OpenAI chat completions messages. + +Your goal is to analyze several LLM runs to find what are the 'static' parts of the instructions and what are the 'dynamic' parts of the instructions that change from run to run. + +Then you must propose a templated version of the instruction with placeholders to inject the variables surrounded by double curly braces in 'instructions_with_input_variables' + +Also provide a JSON example of the input variables extract from one the 'agent_inputs' in 'input_variables_example'. + +You must not change anything in the 'static' parts of the instructions, only add placeholders for the input variables. +""" + + +@workflowai.agent( + id="input-variables-extractor-agent", + version=workflowai.VersionProperties( + model=workflowai.Model.GEMINI_2_0_FLASH_001, + instructions=INSTRUCTIONS, + ), +) +async def input_variables_extractor_agent(input: InputVariablesExtractorInput) -> InputVariablesExtractorOutput: ... diff --git a/api/core/agents/integration_agent.py b/api/core/agents/integration_agent.py index 7cb7af4ac..552ce32ba 100644 --- a/api/core/agents/integration_agent.py +++ b/api/core/agents/integration_agent.py @@ -5,7 +5,7 @@ from pydantic import BaseModel, Field from core.domain.documentation_section import DocumentationSection -from core.domain.integration_domain import Integration +from core.domain.integration_domain.integration_domain import Integration class IntegrationAgentChatMessage(BaseModel): diff --git a/api/core/agents/meta_agent_proxy.py b/api/core/agents/meta_agent_proxy.py new file mode 100644 index 000000000..204d4ecfc --- /dev/null +++ b/api/core/agents/meta_agent_proxy.py @@ -0,0 +1,600 @@ +import datetime +from typing import Any, AsyncIterator, Literal, Self + +import workflowai +from openai import AsyncOpenAI +from pydantic import BaseModel, Field + +from core.domain.documentation_section import DocumentationSection +from core.domain.feedback import Feedback +from core.domain.integration_domain.integration_domain import Integration +from core.domain.url_content import URLContent +from core.domain.version_environment import VersionEnvironment + +from .extract_company_info_from_domain_task import Product + + +class WorkflowaiPage(BaseModel): + title: str + description: str + + +class WorkflowaiSection(BaseModel): + name: str + pages: list[WorkflowaiPage] + + +# MVP for the redirection feature, will be replaced by a dynamic feature in the future +STATIC_WORKFLOWAI_PAGES = [ + WorkflowaiSection( # noqa: F821 + name="Iterate", + pages=[ + WorkflowaiPage( + title="Schemas", + description="Dedicated to the management of agent schemas, allow to see previous schema versions, etc.", + ), + WorkflowaiPage( + title="Playground", + description="The current page the user is on, allow to run agents, on different models, with different instructions, etc.", + ), + WorkflowaiPage( + title="Versions", + description="Allows to see an history of all previous instructions versions of the current agent, with changelogs between versions, etc.", + ), + WorkflowaiPage( + title="Settings", + description="Allow to rename the current agent, delete it, or make it public. Also allows to manage private keys that allow to run the agent via API / SDK.", + ), + ], + ), + WorkflowaiSection( + name="Compare", + pages=[ + WorkflowaiPage( + title="Reviews", + description="Allows to visualize the annotated output for this agents (positive, negative, etc.)", + ), + WorkflowaiPage( + title="Benchmarks", + description="Allows to compare model correctness, cost, latency, based on a set of reviews.", + ), + ], + ), + WorkflowaiSection( + name="Integrate", + pages=[ + WorkflowaiPage( + title="Code", + description="Get ready-to-use Python SDK code snippets, TypeScript SDK code snippets, and example REST requests to run the agent via API.", + ), + WorkflowaiPage( + title="Deployments", + description="Allows to deploy the current agent to fixed environments 'dev', 'staging', 'production'. This allows, for example,to quickly hotfix instructions in production, since the code point to a static 'production' deployment", + ), + ], + ), + WorkflowaiSection( + name="Monitor", + pages=[ + WorkflowaiPage( + title="User Feedback", + description="Allows to see an history of all previous user feedbacks for the current agent.", + ), + WorkflowaiPage( + title="Runs", + description="Allows to see an history of all previous runs of the current agent. 'Run' refers to a single execution of the agent, with a given input, instructions and a given model.", + ), + WorkflowaiPage( + title="Costs", + description="Allows to visualize the cost incurred by the agent per day, for yesterday, last week, last month, last year, and all time.", + ), + ], + ), +] + + +class BaseResult(BaseModel): + tool_name: str = Field( + description="The name of the tool call", + ) + + status: Literal["assistant_proposed", "user_ignored", "completed", "failed"] = Field( + description="The status of the tool call", + ) + + +class BaseToolCallRequest(BaseModel): + ask_user_confirmation: bool | None = Field( + default=None, + description="Whether the tool call should be automatically executed by on the frontend (ask_user_confirmation=false), or if the user should be prompted to run the tool call (ask_user_confirmation=true). Based on the confidence of the meta-agent in the tool call.", + ) + + +class ImprovePromptToolCallRequest(BaseToolCallRequest): + agent_run_id: str | None = Field( + default=None, + description="The id (agent_runs.id) of the runs among the 'agent_runs' that is the most representative of what we want to improve in the 'agent_instructions'", + ) + instruction_improvement_request_message: str = Field( + description="The feedback on the agent run (what is wrong with the output of the run, what is the expected output, etc.).", + ) + + +class ImprovePromptToolCallResult(BaseResult, ImprovePromptToolCallRequest): + pass + + +class EditSchemaStructureToolCallRequest(BaseToolCallRequest): + edition_request_message: str | None = Field( + default=None, + description="The message to edit the agent schema with.", + ) + + +class EditSchemaDescriptionAndExamplesToolCallRequest(BaseToolCallRequest): + description_and_examples_edition_request_message: str | None = Field( + default=None, + description="The message to edit the agent schema's fields description and examples with.", + ) + + +class EditSchemaToolCallResult(BaseResult, EditSchemaStructureToolCallRequest): + pass + + +class RunCurrentAgentOnModelsToolCallRequest(BaseToolCallRequest): + class RunConfig(BaseModel): + run_on_column: Literal["column_1", "column_2", "column_3"] | None = Field( + default=None, + description="The column to run the agent on the agent will be run on all columns", + ) + model: str | None = Field( + default=None, + description="The model to run the agent on the agent will be run on all models", + ) + + run_configs: list[RunConfig] | None = Field( + default=None, + description="The list of configurations to run the current agent on.", + ) + + +class RunCurrentAgentOnModelsToolCallResult(BaseResult, RunCurrentAgentOnModelsToolCallRequest): + pass + + +class GenerateAgentInputToolCallRequest(BaseToolCallRequest): + instructions: str | None = Field( + default=None, + description="The instructions on how to generate the agent input, this message will be passed to the input generation agent.", + ) + + +class GenerateAgentInputToolCallResult(BaseResult, GenerateAgentInputToolCallRequest): + pass + + +class ProxyMetaAgentChatMessage(BaseModel): + role: Literal["USER", "PLAYGROUND", "ASSISTANT"] = Field( + description="The role of the message sender, 'USER' is the actual human user browsing the playground, 'PLAYGROUND' are automated messages sent by the playground to the agent, and 'ASSISTANT' being the assistant generated by the agent", + ) + content: str = Field( + description="The content of the message", + examples=[ + "Thank you for your help!", + "What is the weather forecast for tomorrow?", + ], + ) + + +class PlaygroundState(BaseModel): + class Agent(BaseModel): + name: str + slug: str + schema_id: int + description: str | None = None + input_schema: dict[str, Any] + output_schema: dict[str, Any] + used_integration: Integration | None = None + + current_agent: Agent = Field( + description="The current agent to use for the conversation", + ) + + """ + agent_input: dict[str, Any] | None = Field( + default=None, + description="The input for the agent", + ) + + class InputFile(BaseModel): + key_path: str + file: File + + agent_input_files: list[InputFile] | None = Field( + default=None, + description="The files contained in the 'agent_input' object, if any", + ) + + agent_instructions: str | None = Field( + default=None, + description="The instructions for the agent", + ) + agent_temperature: float | None = Field( + default=None, + description="The temperature for the agent", + ) + """ + + class AgentRun(BaseModel): + id: str = Field( + description="The id of the agent run", + ) + model: str | None = Field( + default=None, + description="The model that was used to generate the agent output", + ) + output: str | None = Field( + default=None, + description="The output of the agent, if no error occurred.", + ) + error: dict[str, Any] | None = Field( + default=None, + description="The error that occurred during the agent run, if any.", + ) + raw_run_request: dict[str, Any] | None = Field( + default=None, + description="The raw run request that was used to generate the agent output", + ) + + class ToolCall(BaseModel): + name: str + input: dict[str, Any] + + tool_calls: list[ToolCall] | None = Field( + default=None, + description="The tool calls that were made by the agent to produce the output", + ) + cost_usd: float | None = Field( + default=None, + description="The cost of the agent run in USD", + ) + duration_seconds: float | None = Field( + default=None, + description="The duration of the agent in seconds", + ) + user_evaluation: Literal["positive", "negative"] | None = Field( + default=None, + description="The user evaluation of the agent output", + ) + + class PlaygroundModel(BaseModel): + id: str = Field( + description="The id of the model", + ) + name: str + is_default: bool = Field( + default=False, + description="Whether the model is one of the default models on the WorkflowAI platform", + ) + is_latest: bool = Field( + default=False, + description="Whether the model is the latest model in its family", + ) + quality_index: int = Field( + description="The quality index that quantifies the reasoning abilities of the model", + ) + context_window_tokens: int = Field( + description="The context window of the model in tokens", + ) + is_not_supported_reason: str | None = Field( + default=None, + description="The reason why the model is not supported for the current agent", + ) + estimate_cost_per_thousand_runs_usd: float | None = Field( + default=None, + description="The estimated cost per thousand runs in USD", + ) + + available_models: list[PlaygroundModel] = Field( + description="The models currently available in the playground", + ) + + """ + class SelectedModels(BaseModel): + column_1: str | None = Field( + default=None, + description="The id of the model selected in the first column of the playground, if empty, no model is selected in the first column", + ) + column_2: str | None = Field( + default=None, + description="The id of the model selected in the second column of the playground, if empty, no model is selected in the second column", + ) + column_3: str | None = Field( + default=None, + description="The id of the model selected in the third column of the playground, if empty, no model is selected in the third column", + ) + + selected_models: SelectedModels = Field( + description="The models currently selected in the playground", + ) + """ + agent_runs: list[AgentRun] | None = Field( + default=None, + description="The agent runs", + ) + + +class ProxyMetaAgentInput(BaseModel): + current_datetime: datetime.datetime = Field( + description="The current datetime", + ) + + messages: list[ProxyMetaAgentChatMessage] = Field( + description="The list of messages in the conversation, the last message being the most recent one", + ) + + latest_messages_url_content: list[URLContent] = Field( + default_factory=list, + description="The URL content of the latest 'USER' message, if any URL was found in the message.", + ) + + class CompanyContext(BaseModel): + company_name: str | None = None + company_description: str | None = None + company_locations: list[str] | None = None + company_industries: list[str] | None = None + company_products: list[Product] | None = None + existing_agents_descriptions: list[str] | None = Field( + default=None, + description="The list of existing agents for the company", + ) + + company_context: CompanyContext = Field( + description="The context of the company to which the conversation belongs", + ) + + workflowai_sections: list[WorkflowaiSection] = Field( + default=STATIC_WORKFLOWAI_PAGES, + description="Other sections pages of the WorkflowAI platform (outside of the playground page, which this agent is part of). You can use this information to answer questions about the WorkflowAI platform and direct the user to the relevant pages. All those page are clickable on the left panel from the WorkflowAI playground.", + ) + + workflowai_documentation_sections: list[DocumentationSection] = Field( + description="The relevant documentation sections of the WorkflowAI platform, which this agent is part of", + ) + + integration_documentation: list[DocumentationSection] = Field( + description="The documentation of the integration that the user is using, if any", + ) + + available_tools_description: str = Field( + description="The description of the available tools, that can be potientially added to the 'agent_instructions' in order to improve the agent's output", + ) + + playground_state: PlaygroundState + + class AgentLifecycleInfo(BaseModel): + class DeploymentInfo(BaseModel): + has_api_or_sdk_runs: bool | None = Field( + default=None, + description="Whether the 'current_agent' has already been run via API / SDK", + ) + latest_api_or_sdk_run_date: datetime.datetime | None = Field( + default=None, + description="The date of the latest API / SDK run", + ) + + class Deployment(BaseModel): + deployed_at: datetime.datetime | None = Field( + default=None, + description="The date of the deployment", + ) + deployed_by_email: str | None = Field( + default=None, + description="The email of the staff member who deployed the 'current_agent' version", + ) + environment: VersionEnvironment | None = Field( + default=None, + description="The environment in which the 'current_agent' version is deployed ('dev', 'staging' or 'production')", + ) + model_used: str | None = Field( + default=None, + description="The model used to run the 'current_agent' deployment", + ) + last_active_at: datetime.datetime | None = Field( + default=None, + description="The date of the last run of the 'current_agent' deployment", + ) + run_count: int | None = Field( + default=None, + description="The number of runs of the 'current_agent' deployment", + ) + notes: str | None = Field( + default=None, + description="The notes of the 'current_agent' deployment, added by the staff member who created the deployed version", + ) + + deployments: list[Deployment] | None = Field( + default=None, + description="The list of deployments of the 'current_agent'", + ) + + deployment_info: DeploymentInfo | None = Field( + default=None, + description="The deployment info of the agent", + ) + + class FeedbackInfo(BaseModel): + user_feedback_count: int | None = Field( + default=None, + description="The number of user feedbacks", + ) + + class AgentFeedback(BaseModel): + created_at: datetime.datetime | None = None + outcome: Literal["positive", "negative"] | None = None + comment: str | None = None + + @classmethod + def from_domain(cls, feedback: Feedback) -> Self: + return cls( + created_at=feedback.created_at, + outcome=feedback.outcome, + comment=feedback.comment, + ) + + latest_user_feedbacks: list[AgentFeedback] | None = Field( + default=None, + description="The 10 latest user feedbacks", + ) + + feedback_info: FeedbackInfo | None = Field( + default=None, + description="The info related to the user feedbacks of the agent.", + ) + + class InternalReviewInfo(BaseModel): + reviewed_input_count: int | None = Field( + default=None, + description="The number of reviewed inputs", + ) + + internal_review_info: InternalReviewInfo | None = Field( + default=None, + description="The info related to the internal reviews of the agent.", + ) + + agent_lifecycle_info: AgentLifecycleInfo | None = Field( + default=None, + description="The lifecycle info of the agent", + ) + + suggested_instructions_with_input_variables: str | None = Field( + default=None, + description="The suggested instructions with input variables, if any", + ) + suggested_input_variables_example: dict[str, Any] | None = Field( + default=None, + description="The suggested input variables example, if any", + ) + + suggested_output_class_code: str | None = Field( + default=None, + description="The suggested output class code to migrate to structured generation, if any", + ) + suggested_instructions_parts_to_remove: list[str] | None = Field( + default=None, + description="The suggested instructions parts to remove from the original instructions when migrating to structured generation, if any", + ) + + +class ProxyMetaAgentOutput(BaseModel): + assistant_answer: str | None = Field( + default=None, + description="The content of the answer message from the meta-agent", + ) + + +_PROXY_META_AGENT_COMMON_INSTRUCTIONS = """Your WorkflowAI proxy playground agent's role is to make the user succeed in the WorkflowAI platform, having performant and reliable agents. + + +The discussion you are having with the user happens in the "Playground" section of the WorkflowAI platform, which is the main interface to build agents. + +It's very important that you check the 'playground_state.agent_runs' to see the latest agent runs that the user has made. +Pay attention to the temporality of those runs, with newer runs being at the beginning of the agent runs array. + +If the first run in the array use Claude models, and the second one is GPT, that means the user has switched to Claude. If there are back and forth between models, consider that the user has tried cloud but went back to GPT. + +When returning code blocks, always make sure the code block is customized to the user's specific context—for example, the model it uses or the name of its agent ('current_agent.slug'). +When returning code blocks, always make sure to just return the part that is to be updated and do not return a full working code block that is too lengthy. +When returning code blocks, do not use diff formate (+, -, etc.) because the user will not be able to copy paste the code block effectively. +Inside returning code blocks do not forget to escape double quotes: \" , same for triple quotes: \"\"\" +Always refer to the 'integration_documentation' that matchers 'playground_state.current_agent.used_integration' for code block language and syntax. +""" + +GENERIC_PROPOSED_INSTRUCTIONS_INSTRUCTIONS = f""" +{_PROXY_META_AGENT_COMMON_INSTRUCTIONS} + +# 1. Trying non-OpenAI models +- Check if the user is using OpenAI models, for example GPT-4, GPT-4 Mini, O1, O3, O4, etc. And if so, you can suggest they try other models from providers Claude, Llama, Grok. You can check all models in the 'available_models' section of the playground state. + +The reason to use different models than OpenAI is having better performing models, cheaper models, faster models. + +If you detect that the user is still using OpenAI models, you can suggest them to switch try different models by first trying the models in the Playground and then modifying the model string in their code (as explained in the instruction documentation). + +# 2. Suggest to use input variables + +As mentioned in the WorkflowAI documentation, using input variables has a lot of advantages when possible. It allows for observability and other features, and also opens the door for deployments. + +## How to spot if the user is using input variables or not ? + +You can know that the user is using input variables if you see variables names surrounded by double curly braces. +If using input variables, the actual variable values will need to be passed in the 'input' payload of the 'raw_run_request'. So, you will need to double-check that as well because if the user omits passing variable values in the 'raw_run_request.input', the agent will not work as expected. + +So if the user is not using yet input variables or badly, you need to suggest him to make the update. Inform your argumentation based on what you will find in the documentation. Note that the exact way of injecting input variables depends on the integration used, but is often by passing 'extra_body'...'. + +Please be aware that your suggestions are optional for the user and if the user declines to use a non-OpenAI model, don't bother them with that again. You can suggest them the next step which is using the input variables. And so on with the structured generation. + +You must check that the user is leveraging WorkflowAI's feature for his agent in the order below (1., 2., 3.). After checking each criteria, you can go to the next one if the user is already using this feature. Do not mention the steps that are solved already. +""" + +PROPOSE_NON_OPENAI_MODELS_INSTRUCTIONS = f""" +{_PROXY_META_AGENT_COMMON_INSTRUCTIONS} + +Your precise goal here is to make the user migrate to non-OpenAI models. You can check the documentation to see why it's a good idea to use non-OpenAI models. + +The reason to use different models than OpenAI is having better performing models, cheaper models, faster models. + +You must pick a relevant model for the use case of the client and show him some code blocks that show how to update the model. +""" + +PROPOSE_INPUT_VARIABLES_INSTRUCTIONS = f""" +{_PROXY_META_AGENT_COMMON_INSTRUCTIONS} + +Your precise goal here is to make the user migrate to input variables. You can check the documentation to see why it's a good idea to use input variables. +Use the 'suggested_instructions_with_input_variables' and 'suggested_input_variables_example' in order to provide code snippet that are valid in the context of the WorkflowAI integrations documented in the 'workflowai_documentation_sections' section of the input.` + +Start by 1. Explaining to the user why it's a good idea to use input variables. Start with a simple phrase like "I have something to propose to make your to unlock a lot of WorkflowAI's capabilities: ..." +Then 2. ALWAYS return the WHOLE 'suggested_instructions_with_input_variables' in a code code block in order for the user to easily copy paste it. +ALWAYS provide a block that shows how to pass the input variables in the completion request (with extra_body=...) +When instructions are spread over several messages, make sure to display a code block that showed several messages with the right instructions at the right place. +""" + +PROPOSE_STRUCTURED_OUTPUT_INSTRUCTIONS = f""" +{_PROXY_META_AGENT_COMMON_INSTRUCTIONS} + +Your precise goal here is to make the user migrate to structured output. You can check the documentation to see why it's a good idea to use structured output. +Use the 'suggested_output_class_code' and 'suggested_instructions_parts_to_remove' in order to provide code snippet that are valid in the context of the WorkflowAI integrations documented in the 'workflowai_documentation_sections' section of the input.` +If 'suggested_instructions_parts_to_remove' are fed, you can mention to the user that they can remove those parts in their existing instructions that talk about generating a valid JSON, because this is not needed anymore when you use structure generation. + +Start by 1. Explaining to the user why it's a good idea to use structured output. Start with a simple phrase like "Next step is to migrate to structured output...." +Then 2. Provide the code snippets to the user that will allow them to use structured output. Based on 'suggested_output_class_code' and how to plug the right respons format in the code. +Also add the imports in the code block. + +If you mention a SDK or package, etc., make sure you are mentioning the right one, for example "Instructor". You do not need to rebuild the full code snippets, just higlight the main changes to do and a few lines of code around it. + +Be aware that the user can just update his code and has nothing to do in the interface. Just updating the code is enough, and a new schema and the agent will be automatically updated in WorkflowAI.com. +""" + + +@workflowai.agent( + id="proxy-meta-agent", +) +async def proxy_meta_agent(_: ProxyMetaAgentInput) -> ProxyMetaAgentOutput: ... + + +async def proxy_meta_agent_proxy(input: ProxyMetaAgentInput, instructions: str) -> AsyncIterator[ProxyMetaAgentOutput]: + client = AsyncOpenAI( + api_key="wai-4hcqxDO3eZLytkIsLdSHGLbviAP0P16bRoX6AVGLTFM", + base_url="https://run.workflowai.dev/v1", + ) + response = await client.chat.completions.create( + model="proxy_meta_agent/claude-3-7-sonnet-20250219", + messages=[ + {"role": "system", "content": instructions}, + {"role": "user", "content": "{% raw %}" + input.model_dump_json() + "{% endraw %}"}, + ], + stream=True, + ) + async for chunk in response: + yield ProxyMetaAgentOutput(assistant_answer=chunk.choices[0].delta.content) diff --git a/api/core/agents/output_schema_extractor_agent.py b/api/core/agents/output_schema_extractor_agent.py new file mode 100644 index 000000000..b6cf0169d --- /dev/null +++ b/api/core/agents/output_schema_extractor_agent.py @@ -0,0 +1,41 @@ +from typing import Any + +import workflowai +from pydantic import BaseModel + + +class OutputSchemaExtractorInput(BaseModel): + class AgentRun(BaseModel): + raw_messages: list[list[dict[str, Any]]] + input: str + output: str + + agent_runs: list[AgentRun] + programming_language: str + structured_object_class: str + + +class OutputSchemaExtractorOutput(BaseModel): + proposed_structured_object_class: str + instructions_parts_to_remove: list[str] + + +INSTRUCTIONS = """ +You are an expert at extracting output schema from agent contained in OpenAI chat completions messages. + +Your goal is to analyze several LLM runs to find what is the output schema of the agent. + +You must propose a class that implements 'structured_object_class' that will be used as the output schema of the agent. Please always included the required imports in the code block. + +You must also eventually remove some instructions from the original instructions that does not make sense to when using structured generations (ex: "You must return a JSON object"). +""" + + +@workflowai.agent( + id="output-schema-extractor-agent", + version=workflowai.VersionProperties( + model=workflowai.Model.GEMINI_2_0_FLASH_001, + instructions=INSTRUCTIONS, + ), +) +async def output_schema_extractor_agent(input: OutputSchemaExtractorInput) -> OutputSchemaExtractorOutput: ... diff --git a/api/core/domain/integration_domain.py b/api/core/domain/integration_domain.py deleted file mode 100644 index 0b3ccb2ab..000000000 --- a/api/core/domain/integration_domain.py +++ /dev/null @@ -1,109 +0,0 @@ -from enum import Enum - -from pydantic import BaseModel - -WORKFLOWAI_API_KEY_PLACEHOLDER = "" -PROPOSED_AGENT_NAME_AND_MODEL_PLACEHOLDER = "" - - -class IntegrationPartner(str, Enum): - INSTRUCTOR = "instructor" - OPENAI_SDK = "openai-sdk" - - -class ProgrammingLanguage(str, Enum): - PYTHON = "python" - TYPESCRIPT = "typescript" - - -class IntegrationKind(str, Enum): - INSTRUCTOR_PYTHON = "instructor-python" - - -class Integration(BaseModel): - integration_partner: IntegrationPartner - programming_language: ProgrammingLanguage - display_name: str - slug: IntegrationKind - - logo_url: str - landing_page_snippet: str - - # Not all integration may support structured generation. - landing_page_structured_generation_snippet: str | None = None - - integration_chat_initial_snippet: str - integration_chat_agent_naming_snippet: str - - documentation_filepaths: list[str] - - -INSTRUCTOR_PYTHON_LANDING_PAGE_SNIPPET = """import os - -import instructor -from openai import OpenAI -from pydantic import BaseModel - - -class UserInfo(BaseModel): - name: str - age: int - -def extract_user_info(user_message: str) -> UserInfo: - client = instructor.from_openai( - OpenAI(base_url="https://run.workflowai.com/v1", api_key=""), - mode=instructor.Mode.OPENROUTER_STRUCTURED_OUTPUTS, - ) - - return client.chat.completions.create( - model="user-info-extraction-agent/claude-3-7-sonnet-latest", # Agent now runs Claude 3.7 Sonnet - response_model=UserInfo, - messages=[{"role": "user", "content": user_message}], - ) - -if __name__ == "__main__": - user_info = extract_user_info("John Black is 33 years old.") - print("Basic example result:", user_info) # UserInfo(name='John Black', age=33)""" - -INSTRUCTOR_PYTHON_LANDING_PAGE_STRUCTURED_GENERATION_SNIPPET = INSTRUCTOR_PYTHON_LANDING_PAGE_SNIPPET - -INSTRUCTOR_PYTHON_INTEGRATION_CHAT_INITIAL_SNIPPET = """import instructor -from openai import OpenAI - -# After (WorkflowAI Proxy) -client = instructor.from_openai( - OpenAI( - base_url="https://run.workflowai.com/v1", # OpenAI now uses WorkflowAI's URL and API key - api_key= - ), - mode=instructor.Mode.OPENROUTER_STRUCTURED_OUTPUTS, -) - - -# Everything else (model calls, parameters) stays the same -response = client.chat.completions.create( - ..., -) -""" - -INSTRUCTOR_PYTHON_INTEGRATION_CHAT_AGENT_NAMING_SNIPPET = """response = client.chat.completions.create( - model="[", # Or claude-3-7-sonnet-latest - messages=[{"role": "user", "content": "Hello!"}] -)""" - -OFFICIAL_INTEGRATIONS = [ - Integration( - integration_partner=IntegrationPartner.INSTRUCTOR, - programming_language=ProgrammingLanguage.PYTHON, - display_name="Instructor (Python)", - slug=IntegrationKind.INSTRUCTOR_PYTHON, - logo_url="https://workflowai.blob.core.windows.net/workflowai-public/python.png", - landing_page_snippet=INSTRUCTOR_PYTHON_LANDING_PAGE_SNIPPET, - landing_page_structured_generation_snippet=INSTRUCTOR_PYTHON_LANDING_PAGE_STRUCTURED_GENERATION_SNIPPET, - integration_chat_initial_snippet=INSTRUCTOR_PYTHON_INTEGRATION_CHAT_INITIAL_SNIPPET, - integration_chat_agent_naming_snippet=INSTRUCTOR_PYTHON_INTEGRATION_CHAT_AGENT_NAMING_SNIPPET, - documentation_filepaths=[ - "developers/python/instructor.md", - ], - ), -] diff --git a/api/core/domain/integration_domain/instruction_python_snippets.py b/api/core/domain/integration_domain/instruction_python_snippets.py new file mode 100644 index 000000000..8558f35fa --- /dev/null +++ b/api/core/domain/integration_domain/instruction_python_snippets.py @@ -0,0 +1,52 @@ +INSTRUCTOR_PYTHON_LANDING_PAGE_SNIPPET = """import os + +import instructor +from openai import OpenAI +from pydantic import BaseModel + + +class UserInfo(BaseModel): + name: str + age: int + +def extract_user_info(user_message: str) -> UserInfo: + client = instructor.from_openai( + OpenAI(base_url="https://run.workflowai.com/v1", api_key=""), + mode=instructor.Mode.OPENROUTER_STRUCTURED_OUTPUTS, + ) + + return client.chat.completions.create( + model="user-info-extraction-agent/claude-3-7-sonnet-latest", # Agent now runs Claude 3.7 Sonnet + response_model=UserInfo, + messages=[{"role": "user", "content": user_message}], + ) + +if __name__ == "__main__": + user_info = extract_user_info("John Black is 33 years old.") + print("Basic example result:", user_info) # UserInfo(name='John Black', age=33)""" + +INSTRUCTOR_PYTHON_LANDING_PAGE_STRUCTURED_GENERATION_SNIPPET = INSTRUCTOR_PYTHON_LANDING_PAGE_SNIPPET + +INSTRUCTOR_PYTHON_INTEGRATION_CHAT_INITIAL_SNIPPET = """import instructor +from openai import OpenAI + +# After (WorkflowAI Proxy) +client = instructor.from_openai( + OpenAI( + base_url="https://run.workflowai.com/v1", # OpenAI now uses WorkflowAI's URL and API key + api_key= + ), + mode=instructor.Mode.OPENROUTER_STRUCTURED_OUTPUTS, +) + + +# Everything else (model calls, parameters) stays the same +response = client.chat.completions.create( + ..., +) +""" + +INSTRUCTOR_PYTHON_INTEGRATION_CHAT_AGENT_NAMING_SNIPPET = """response = client.chat.completions.create( + model="[", # Or claude-3-7-sonnet-latest + messages=[{"role": "user", "content": "Hello!"}] +)""" diff --git a/api/core/domain/integration_domain/integration_domain.py b/api/core/domain/integration_domain/integration_domain.py new file mode 100644 index 000000000..d58577216 --- /dev/null +++ b/api/core/domain/integration_domain/integration_domain.py @@ -0,0 +1,121 @@ +from enum import Enum + +from pydantic import BaseModel + +from core.domain.integration_domain.instruction_python_snippets import ( + INSTRUCTOR_PYTHON_INTEGRATION_CHAT_AGENT_NAMING_SNIPPET, + INSTRUCTOR_PYTHON_INTEGRATION_CHAT_INITIAL_SNIPPET, + INSTRUCTOR_PYTHON_LANDING_PAGE_SNIPPET, + INSTRUCTOR_PYTHON_LANDING_PAGE_STRUCTURED_GENERATION_SNIPPET, +) +from core.domain.integration_domain.openai_sdk_python_snippets import ( + OPENAI_SDK_PYTHON_INTEGRATION_CHAT_AGENT_NAMING_SNIPPET, + OPENAI_SDK_PYTHON_INTEGRATION_CHAT_INITIAL_SNIPPET, + OPENAI_SDK_PYTHON_LANDING_PAGE_SNIPPET, + OPENAI_SDK_PYTHON_LANDING_PAGE_STRUCTURED_GENERATION_SNIPPET, +) +from core.domain.integration_domain.openai_sdk_ts_snippets import ( + OPENAI_SDK_TS_INTEGRATION_CHAT_AGENT_NAMING_SNIPPET, + OPENAI_SDK_TS_INTEGRATION_CHAT_INITIAL_SNIPPET, + OPENAI_SDK_TS_LANDING_PAGE_SNIPPET, + OPENAI_SDK_TS_LANDING_PAGE_STRUCTURED_GENERATION_SNIPPET, +) + +WORKFLOWAI_API_KEY_PLACEHOLDER = "" +PROPOSED_AGENT_NAME_AND_MODEL_PLACEHOLDER = "" + + +class IntegrationPartner(str, Enum): + INSTRUCTOR = "instructor" + OPENAI_SDK = "openai-sdk" + OPENAI_SDK_TS = "openai-sdk-ts" + + +class ProgrammingLanguage(str, Enum): + PYTHON = "python" + TYPESCRIPT = "typescript" + + +class IntegrationKind(str, Enum): + INSTRUCTOR_PYTHON = "instructor-python" + OPENAI_SDK_PYTHON = "openai-sdk-python" + OPENAI_SDK_TS = "openai-sdk-ts" + + +class Integration(BaseModel): + integration_partner: IntegrationPartner + programming_language: ProgrammingLanguage + default_for_language: bool + output_class: str + display_name: str + slug: IntegrationKind + + logo_url: str + landing_page_snippet: str + + # Not all integration may support structured generation. + landing_page_structured_generation_snippet: str | None = None + + integration_chat_initial_snippet: str + integration_chat_agent_naming_snippet: str + + documentation_filepaths: list[str] + + +OFFICIAL_INTEGRATIONS = [ + Integration( + integration_partner=IntegrationPartner.INSTRUCTOR, + programming_language=ProgrammingLanguage.PYTHON, + default_for_language=False, + output_class="pydantic.BaseModel", + display_name="Instructor (Python)", + slug=IntegrationKind.INSTRUCTOR_PYTHON, + logo_url="https://workflowai.blob.core.windows.net/workflowai-public/python.png", + landing_page_snippet=INSTRUCTOR_PYTHON_LANDING_PAGE_SNIPPET, + landing_page_structured_generation_snippet=INSTRUCTOR_PYTHON_LANDING_PAGE_STRUCTURED_GENERATION_SNIPPET, + integration_chat_initial_snippet=INSTRUCTOR_PYTHON_INTEGRATION_CHAT_INITIAL_SNIPPET, + integration_chat_agent_naming_snippet=INSTRUCTOR_PYTHON_INTEGRATION_CHAT_AGENT_NAMING_SNIPPET, + documentation_filepaths=[ + "developers/python/instructor.md", + ], + ), + Integration( + integration_partner=IntegrationPartner.OPENAI_SDK, + programming_language=ProgrammingLanguage.PYTHON, + default_for_language=True, + output_class="pydantic.BaseModel", + display_name="OpenAI SDK (Python)", + slug=IntegrationKind.OPENAI_SDK_PYTHON, + logo_url="https://workflowai.blob.core.windows.net/workflowai-public/python.png", + landing_page_snippet=OPENAI_SDK_PYTHON_LANDING_PAGE_SNIPPET, + landing_page_structured_generation_snippet=OPENAI_SDK_PYTHON_LANDING_PAGE_STRUCTURED_GENERATION_SNIPPET, + integration_chat_initial_snippet=OPENAI_SDK_PYTHON_INTEGRATION_CHAT_INITIAL_SNIPPET, + integration_chat_agent_naming_snippet=OPENAI_SDK_PYTHON_INTEGRATION_CHAT_AGENT_NAMING_SNIPPET, + documentation_filepaths=[ + "developers/python/openai.md", + ], + ), + Integration( + integration_partner=IntegrationPartner.OPENAI_SDK, + programming_language=ProgrammingLanguage.TYPESCRIPT, + default_for_language=True, + output_class="zod.z.object", + display_name="OpenAI SDK (TypeScript)", + slug=IntegrationKind.OPENAI_SDK_TS, + logo_url="https://workflowai.blob.core.windows.net/workflowai-public/typescript.png", + landing_page_snippet=OPENAI_SDK_TS_LANDING_PAGE_SNIPPET, + landing_page_structured_generation_snippet=OPENAI_SDK_TS_LANDING_PAGE_STRUCTURED_GENERATION_SNIPPET, + integration_chat_initial_snippet=OPENAI_SDK_TS_INTEGRATION_CHAT_INITIAL_SNIPPET, + integration_chat_agent_naming_snippet=OPENAI_SDK_TS_INTEGRATION_CHAT_AGENT_NAMING_SNIPPET, + documentation_filepaths=[ + "developers/js/openai.md", + ], + ), +] + + +def default_integration_for_language(language: ProgrammingLanguage) -> Integration: + for integration in OFFICIAL_INTEGRATIONS: + if integration.programming_language == language and integration.default_for_language: + return integration + raise ValueError(f"No default integration found for language: {language}") diff --git a/api/core/domain/integration_domain/openai_sdk_python_snippets.py b/api/core/domain/integration_domain/openai_sdk_python_snippets.py new file mode 100644 index 000000000..db74ce7cb --- /dev/null +++ b/api/core/domain/integration_domain/openai_sdk_python_snippets.py @@ -0,0 +1,73 @@ +OPENAI_SDK_PYTHON_LANDING_PAGE_SNIPPET = """import os + +import openai + +# Configure the OpenAI client to use the WorkflowAI endpoint and API key +openai.api_key = os.environ.get("WORKFLOWAI_API_KEY") # Use your WorkflowAI API key +openai.api_base = "https://run.workflowai.com/v1" + +response = openai.ChatCompletion.create( + model="gpt-4o-2024-11-20", # Or any model supported by your WorkflowAI setup + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello!"}, + ], +) + +print(response.choices[0].message.content)""" + +# For integrations that showcase structured generation we reuse the landing page snippet +# In future this can be swapped for a dedicated structured output example using `.parse()`. +OPENAI_SDK_PYTHON_LANDING_PAGE_STRUCTURED_GENERATION_SNIPPET = """import os + +import openai +from pydantic import BaseModel + + +# Configure the OpenAI client to use the WorkflowAI endpoint and API key +client = openai.OpenAI( + api_key=os.environ.get("WORKFLOWAI_API_KEY"), # Use your WorkflowAI API key + base_url="https://run.workflowai.com/v1", +) + + +class CountryInfo(BaseModel): + country: str + + +def get_country(city: str) -> CountryInfo: + # Return the country of a city, parsed as a Pydantic object. + + completion = client.beta.chat.completions.parse( + # Always prefix the model with an agent name for clear organization in WorkflowAI + model="country-extractor/gpt-4o-2024-11-20", + messages=[ + { + "role": "system", + "content": "You are a helpful assistant that extracts geographical information.", + }, + {"role": "user", "content": f"What is the country of {city}?"}, + ], + # Pass the Pydantic class directly as the response format + response_format=CountryInfo, + ) + + # Access the parsed Pydantic object directly + return completion.choices[0].message.parsed +""" + +OPENAI_SDK_PYTHON_INTEGRATION_CHAT_INITIAL_SNIPPET = """import openai + +# After (WorkflowAI Proxy) +openai.api_key = +openai.api_base = "https://run.workflowai.com/v1" + +# Everything else (model calls, parameters) stays the same +response = openai.ChatCompletion.create( + ..., +)""" + +OPENAI_SDK_PYTHON_INTEGRATION_CHAT_AGENT_NAMING_SNIPPET = """response = openai.ChatCompletion.create( + model="[", # e.g. my-agent/gpt-4o-2024-11-20 + messages=[{"role": "user", "content": "Hello!"}] +)""" diff --git a/api/core/domain/integration_domain/openai_sdk_ts_snippets.py b/api/core/domain/integration_domain/openai_sdk_ts_snippets.py new file mode 100644 index 000000000..6e4baf1d4 --- /dev/null +++ b/api/core/domain/integration_domain/openai_sdk_ts_snippets.py @@ -0,0 +1,92 @@ +# --------------------------------------------------------------------------- +# OpenAI SDK (TypeScript/JavaScript) – WorkflowAI Integration Code Snippets +# --------------------------------------------------------------------------- +# These snippets are embedded in the public documentation and within the in-app +# onboarding flow. They MUST be valid TypeScript / modern JavaScript so that +# developers can copy-paste them directly into a Create-React-App / Node / Bun +# or Vite project without modification. + + +# Landing Page (Basic Usage) -------------------------------------------------- +OPENAI_SDK_TS_LANDING_PAGE_SNIPPET = """import OpenAI from 'openai'; + +// 1. Configuration – point the OpenAI SDK at WorkflowAI +const client = new OpenAI({ + apiKey: process.env.WORKFLOWAI_API_KEY || 'YOUR_WORKFLOWAI_API_KEY', + baseURL: 'https://run.workflowai.com/v1', +}); + +// 2. Simple chat request +const response = await client.chat.completions.create({ + // Always prefix the model with an agent name for better organisation in WorkflowAI + model: 'quick-start-agent/gpt-4o-mini', + messages: [ + { role: 'system', content: 'You are a helpful assistant.' }, + { role: 'user', content: 'Hello!' }, + ], +}); + +console.log(response.choices[0].message.content); +""" + + +# Landing Page (Reliable Structured Output) ----------------------------------- +# Requires openai@^4.55.0 for `beta.chat.completions.parse`. +OPENAI_SDK_TS_LANDING_PAGE_STRUCTURED_GENERATION_SNIPPET = """import OpenAI from 'openai'; +import { z } from 'zod'; +import { zodResponseFormat } from 'openai/helpers/zod'; + +// 1. Configuration – WorkflowAI proxy +const client = new OpenAI({ + apiKey: process.env.WORKFLOWAI_API_KEY || 'YOUR_WORKFLOWAI_API_KEY', + baseURL: 'https://run.workflowai.com/v1', +}); + +// 2. Define a Zod schema for structured output +const CountryInfoSchema = z.object({ + country: z.string().describe('The country where the city is located.'), + continent: z.string().describe('The continent of the country.'), +}); + +// 3. Function wrapping a structured request +export async function getCountryInfo(city: string) { + const completion = await client.beta.chat.completions.parse({ + model: 'geo-extractor/gpt-4o', + messages: [ + { role: 'system', content: 'You are a helpful assistant that provides geographical information.' }, + { role: 'user', content: `Which country is ${city} in and on which continent?` }, + ], + // Validate & parse against the Zod schema + response_format: zodResponseFormat(CountryInfoSchema, 'country_info'), + }); + + // The SDK returns a parsed object in `message.parsed` + return completion.choices[0].message.parsed; +} +""" + + +# Integration Chat – Initial "How-To" Message ---------------------------------- +# NOTE: The token is dynamically replaced by +# the backend when sending instructions to the user. Do not rename it. +OPENAI_SDK_TS_INTEGRATION_CHAT_INITIAL_SNIPPET = """import OpenAI from 'openai'; + +// After (WorkflowAI Proxy) +const client = new OpenAI({ + apiKey: , + baseURL: 'https://run.workflowai.com/v1', +}); + +// Everything else (model calls, parameters) stays the same +const response = await client.chat.completions.create({ + ..., +}); +""" + + +# Integration Chat – Suggesting Agent-Prefixed Model Name ---------------------- +OPENAI_SDK_TS_INTEGRATION_CHAT_AGENT_NAMING_SNIPPET = """const response = await client.chat.completions.create({ + model: '[', // e.g. my-agent/gpt-4o + messages: [{ role: 'user', content: 'Hello!' }], +}); +""" diff --git a/api/core/domain/integration_domain_test.py b/api/core/domain/integration_domain_test.py index 5ce633672..7a5a2115e 100644 --- a/api/core/domain/integration_domain_test.py +++ b/api/core/domain/integration_domain_test.py @@ -1,8 +1,9 @@ -from core.domain.integration_domain import ( +from core.domain.integration_domain.integration_domain import ( OFFICIAL_INTEGRATIONS, PROPOSED_AGENT_NAME_AND_MODEL_PLACEHOLDER, WORKFLOWAI_API_KEY_PLACEHOLDER, IntegrationKind, + ProgrammingLanguage, ) @@ -37,3 +38,19 @@ def test_integration_snippets_contain_required_placeholders(): f"Integration '{integration.display_name}' is missing {PROPOSED_AGENT_NAME_AND_MODEL_PLACEHOLDER} " f"in integration_chat_agent_naming_snippet" ) + + +def test_each_language_has_default_integration(): + default_languages = { + integration.programming_language + for integration in OFFICIAL_INTEGRATIONS + if integration.default_for_language is True + } + all_languages = {language for language in ProgrammingLanguage} + assert all_languages == default_languages, ( + "Each programming language must have exactly one default integration. " + "Missing default for: {missing}. Extra defaults for: {extra}" + ).format( + missing=all_languages - default_languages, + extra=default_languages - all_languages, + ) diff --git a/api/core/storage/clickhouse/clickhouse_client.py b/api/core/storage/clickhouse/clickhouse_client.py index bed8dabc4..8e3ca059d 100644 --- a/api/core/storage/clickhouse/clickhouse_client.py +++ b/api/core/storage/clickhouse/clickhouse_client.py @@ -522,19 +522,23 @@ async def run_count_by_agent_uid( ) @override - async def list_runs_since( + async def list_latest_runs( self, - since_date: datetime, + task_uid: int | None = None, + since_date: datetime | None = None, is_active: bool = True, limit: int = 100, ) -> AsyncIterator[AgentRun]: - # TODO: use SerializableTaskRunQuery instead - w = W("created_at_date", type="Date", value=since_date.strftime("%Y-%m-%d"), operator=">=") & W( - "run_uuid", - type="UInt128", - value=id_lower_bound(since_date), - operator=">=", - ) + w = W("tenant_uid", type="UInt32", value=self.tenant_uid) + if task_uid: + w &= W("task_uid", type="UInt32", value=task_uid) + if since_date: + w &= W("created_at_date", type="Date", value=since_date.strftime("%Y-%m-%d"), operator=">=") & W( + "run_uuid", + type="UInt128", + value=id_lower_bound(since_date), + operator=">=", + ) if is_active: w &= W("is_active", type="Boolean", value=is_active) runs = await self._runs( diff --git a/api/core/storage/mongo/partials/mongo_agent_runs.py b/api/core/storage/mongo/partials/mongo_agent_runs.py index c8a283fc6..563962bb8 100644 --- a/api/core/storage/mongo/partials/mongo_agent_runs.py +++ b/api/core/storage/mongo/partials/mongo_agent_runs.py @@ -522,11 +522,12 @@ def list_runs_for_memory_id( raise NotImplementedError() @override - def list_runs_since( + def list_latest_runs( self, - since_date: datetime, + task_uid: int | None = None, + since_date: datetime | None = None, is_active: bool = True, - limit: int = 10, + limit: int = 100, ) -> AsyncIterator[AgentRun]: raise NotImplementedError() diff --git a/api/core/storage/task_run_storage.py b/api/core/storage/task_run_storage.py index d9d9ef0dd..f4c507b8d 100644 --- a/api/core/storage/task_run_storage.py +++ b/api/core/storage/task_run_storage.py @@ -148,9 +148,10 @@ def run_count_by_agent_uid( is_active: bool | None = None, ) -> AsyncIterator[AgentRunCount]: ... - def list_runs_since( + def list_latest_runs( self, - since_date: datetime, + task_uid: int | None = None, + since_date: datetime | None = None, is_active: bool = True, - limit: int = 10, + limit: int = 100, ) -> AsyncIterator[AgentRun]: ... diff --git a/docs/developers/js/openai.md b/docs/developers/js/openai.md new file mode 100644 index 000000000..16d388e93 --- /dev/null +++ b/docs/developers/js/openai.md @@ -0,0 +1,407 @@ +# OpenAI (JavaScript/TypeScript) + +- clarify that the WorkflowAI client is not yet compatible with the the `responses` API from OpenAI. +- what is the minimum SDK version required for `.beta.....parse` + +> 4.55.0 is the minimum version required for `.beta.....parse` + +- show exactly hwo to use in typescript the type for the parse method + +### Example 2: Chatbot Core Logic + +This snippet demonstrates the core logic for a chatbot: maintaining conversation history and getting the next response. + +```typescript +import OpenAI from 'openai'; +import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions'; +// Import zodResponseFormat helper +import { zodResponseFormat } from "openai/helpers/zod"; +import { z } from 'zod'; + +// --- 1. Configuration --- +// Initialize the OpenAI client configured for WorkflowAI. +const client = new OpenAI({ + apiKey: process.env.WORKFLOWAI_API_KEY || 'YOUR_WORKFLOWAI_API_KEY', + baseURL: 'https://run.workflowai.com/v1', +}); + +// --- 2. Define Structured Output Schema for Chatbot --- +// Define a simple schema for the chatbot's response. +const ChatbotResponseSchema = z.object({ + assistant_message: z.string().describe("The chatbot's response to the user."), +}); + +// --- 3. Conversation State --- +// Store the conversation history. Initialize with a system message. +let conversationHistory: ChatCompletionMessageParam[] = [ + { + role: 'system', + content: 'You are a helpful assistant. Keep your responses concise and provide them in the requested JSON format.', // Adjusted system prompt slightly + }, +]; + +// --- 4. Chat Function Definition (Using Structured Output) --- +/** + * Sends the current conversation history plus the new user message to the model, + * requests a structured response, gets the assistant's message, and updates the history. + * + * @param userMessage The message input by the user. + * @returns The assistant's response message string. + */ +async function getChatbotResponseAndUpdateHistory(userMessage: string): Promise { + // Add the user's message to the history for the API call + const currentMessages = [ + ...conversationHistory, + { role: 'user' as const, content: userMessage } + ]; + + try { + console.log('\\nSending messages (expecting structured response):', JSON.stringify(currentMessages, null, 2)); + + // Use .parse() to request and validate structured output + const completion = await client.beta.chat.completions.parse({ + // Model selection: Use agent prefix + chat model ID. + model: 'chatbot/gpt-4o-mini', + messages: currentMessages, + temperature: 0.7, + // Define the expected structured response format using Zod schema + response_format: zodResponseFormat(ChatbotResponseSchema, "chatbot_response"), + }); + + // Extract the message string from the parsed structured object + const assistantResponse = completion.choices[0]?.message?.parsed?.assistant_message; + + if (assistantResponse) { + // Update the main history state ONLY after a successful response + conversationHistory = [ + ...currentMessages, + // Store the plain string message in the history + { role: 'assistant' as const, content: assistantResponse } + ]; + return assistantResponse; + } else { + // Handle cases where parsing might succeed but the expected field is missing + console.error("Parsed object missing 'assistant_message':", completion.choices[0]?.message?.parsed); + return 'Sorry, I received an unexpected response format.'; + } + } catch (error) { + console.error('Error communicating with or parsing response from WorkflowAI:', error); + // Do not update history on error + return 'An error occurred. Please try again.'; + } +} + +/* +// --- Example Usage (Conceptual) --- +// ... (Usage remains the same as before, calling the async function) +*/ +``` + +This example demonstrates the core loop of a chatbot: taking user input, adding it to the history, sending the history to the API, getting a response, adding the response to the history, and repeating. + +---- + +# WorkflowAI Proxy with TypeScript + +WorkflowAI provides an OpenAI-compatible API endpoint, allowing you to seamlessly use WorkflowAI with your existing TypeScript applications built using the official OpenAI SDK or compatible libraries. By simply updating the base URL and API key, your code will leverage WorkflowAI's features. + +This approach offers several advantages: + +* **Minimal Code Change:** Switch to WorkflowAI by updating the `baseURL` and `apiKey`. +* **Leverage Existing SDKs:** Continue using the familiar `openai` Node.js/TypeScript SDK. +* **Rapid Integration:** Quickly start using WorkflowAI's features. +* **Multi-Provider Model Access:** Access 80+ models via a single API. +* **Reliable Structured Output:** Get guaranteed structured data (e.g., using Zod) from any model. +* **Automatic Cost Calculation:** Get estimated costs per request in the response. +* **Enhanced Reliability:** Benefit from automatic model provider fallbacks. +* **Observability Built-in:** Monitor usage, performance, and costs. + +## Setup + +To use the WorkflowAI proxy with the `openai` TypeScript library, configure the client with your WorkflowAI API key and the WorkflowAI base URL. + +```typescript +import OpenAI from 'openai'; + +// Initialize the OpenAI client configured for WorkflowAI. +const client = new OpenAI({ + apiKey: process.env.WORKFLOWAI_API_KEY || 'YOUR_WORKFLOWAI_API_KEY', // Use your WorkflowAI API Key + baseURL: 'https://run.workflowai.com/v1', // Point to the WorkflowAI endpoint +}); + +// You can now use the 'client' object as you normally would with the OpenAI SDK +// Example: Making a simple chat completion request +async function simpleChat() { + try { + const response = await client.chat.completions.create({ + // Use an agent prefix + model ID (see "Organizing Runs" below) + model: "simple-chatbot/gpt-4o-mini", + messages: [ + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "Hello!" }, + ], + }); + console.log(response.choices[0].message.content); + } catch (error) { + console.error("Error making chat completion:", error); + } +} + +// simpleChat(); // Uncomment to run +``` + +**Key Changes:** + +1. **`apiKey`**: Use your `WORKFLOWAI_API_KEY`. +2. **`baseURL`**: Set the `baseURL` to `https://run.workflowai.com/v1`. + +## Organizing Runs with Agent Prefixes + +To better organize runs in the WorkflowAI dashboard, prefix the `model` parameter with an "agent" name followed by a slash (`/`). This associates the run with a specific logical agent. + +```typescript +const response = await client.chat.completions.create({ + // Model prefixed with agent name 'my-chatbot' + model: "my-chatbot/gpt-4o", + messages: [{ role: "user", content: "Tell me about WorkflowAI." }], +}); +``` + +In the WorkflowAI UI, this run will be grouped under the agent `my-chatbot`. If no prefix is provided, it defaults to the `default` agent. + +## Switching Between Models + +WorkflowAI allows you to use models from multiple providers (OpenAI, Anthropic, Google, etc.) through the same API. Switching is as simple as changing the model identifier string. + +```typescript +async function switchModels() { + const messages = [{ role: "user", content: "Explain the concept of LLM RAG." }]; + + // Using OpenAI's GPT-4o via 'my-explainer' agent + const gptResponse = await client.chat.completions.create({ + model: "my-explainer/gpt-4o", + messages: messages, + }); + console.log("GPT-4o says:", gptResponse.choices[0].message.content?.substring(0, 50) + "..."); + + // Switching to Anthropic's Claude 3.5 Sonnet (verify exact ID on workflowai.com/models) + const claudeResponse = await client.chat.completions.create({ + model: "my-explainer/claude-3.5-sonnet-latest", // Simply change the model ID + messages: messages, + }); + console.log("Claude 3.5 Sonnet says:", claudeResponse.choices[0].message.content?.substring(0, 50) + "..."); + + // Switching to Google's Gemini 1.5 Flash (verify exact ID on workflowai.com/models) + const geminiResponse = await client.chat.completions.create({ + model: "my-explainer/gemini-1.5-flash-latest", // Use the appropriate model identifier + messages: messages, + }); + console.log("Gemini 1.5 Flash says:", geminiResponse.choices[0].message.content?.substring(0, 50) + "..."); +} + +// switchModels(); // Uncomment to run +``` +You don't need separate SDKs or API keys for each provider. Find available model IDs at [workflowai.com/models](https://workflowai.com/models). + +## Reliable Structured Outputs (with Zod) + +WorkflowAI guarantees reliable structured output (like JSON parsed into objects) from *any* model when using the `openai` TypeScript library with Zod schema validation. + +1. Define your desired output structure using a `zod` schema. +2. Use the `client.beta.chat.completions.parse()` method. +3. Use the `zodResponseFormat` helper from `openai/helpers/zod` to pass your schema to the `response_format` parameter. +4. Access the parsed, type-safe object directly from `completion.choices[0].message.parsed`. + +```typescript +import OpenAI from 'openai'; +// Import zodResponseFormat helper and Zod itself +import { zodResponseFormat } from "openai/helpers/zod"; +import { z } from 'zod'; + +// --- 1. Configuration --- +const client = new OpenAI({ + apiKey: process.env.WORKFLOWAI_API_KEY || 'YOUR_WORKFLOWAI_API_KEY', + baseURL: 'https://run.workflowai.com/v1', +}); + +// --- 2. Define Structured Output Schema --- +// Example: Extracting user details +const UserInfoSchema = z.object({ + name: z.string().describe("The full name of the user."), + email: z.string().email().describe("The email address of the user."), + department: z.string().optional().describe("The department the user belongs to, if mentioned.") +}).describe("Schema for user information extracted from text."); + +// --- 3. Function Definition --- +/** + * Extracts user information from text using structured output. + * @param text The text containing user information. + * @returns A promise resolving to an object matching UserInfoSchema. + */ +async function extractUserInfo(text: string): Promise> { + try { + const completion = await client.beta.chat.completions.parse({ + model: 'user-extractor/gpt-4o', // Use an appropriate agent/model + messages: [ + { + role: 'system', + content: 'You are an expert at extracting user details (name, email, department) from text. Respond using the provided tool.', + }, + { + role: 'user', + content: `Extract the user details from the following text: ${text}`, + }, + ], + // Define structured response format using Zod schema and the helper + response_format: zodResponseFormat(UserInfoSchema, "user_information"), // "user_information" is a name for the tool/function + }); + + // Return the parsed, type-safe object + return completion.choices[0].message.parsed; + + } catch (error) { + console.error('Error extracting or parsing user info:', error); + throw error; + } +} + +/* +// --- Example Usage (Conceptual) --- +async function runUserExtraction() { + try { + const text1 = "Please contact John Doe at john.doe@example.com regarding the Q3 report."; + const userInfo1 = await extractUserInfo(text1); + console.log("User Info 1:", userInfo1); + // Expected: { name: 'John Doe', email: 'john.doe@example.com' } + + const text2 = "Reach out to Jane Smith from Marketing via jane.s@company.org."; + const userInfo2 = await extractUserInfo(text2); + console.log("User Info 2:", userInfo2); + // Expected: { name: 'Jane Smith', email: 'jane.s@company.org', department: 'Marketing' } + + } catch (e) { + console.error("Failed to extract user info:", e); + } +} +// runUserExtraction(); // Uncomment to run +*/ +``` + +A key benefit is WorkflowAI's compatibility: this method works reliably across **all models** available through the proxy, even those without native structured output support. You don't need to explicitly ask for JSON in the prompt. + +## Prompt Templating (with Structured Output) + +Combine structured output with Jinja2-style templating in your prompts for cleaner code. Pass template variables using the `input` parameter. + +```typescript +import OpenAI from 'openai'; +import { zodResponseFormat } from "openai/helpers/zod"; +import { z } from 'zod'; + +// --- 1. Configuration --- +const client = new OpenAI({ + apiKey: process.env.WORKFLOWAI_API_KEY || 'YOUR_WORKFLOWAI_API_KEY', + baseURL: 'https://run.workflowai.com/v1', +}); + +// --- 2. Define Structured Output Schema --- +const CountryInfoSchema = z.object({ + country: z.string().describe('The country where the city is located.'), + continent: z.string().describe('The continent where the country is located.'), +}); + +// --- 3. Function Definition --- +/** + * Fetches geographical information for a given city using the WorkflowAI proxy, + * prompt templating, and structured output via zodResponseFormat. + * + * @param city The name of the city to query. + * @returns A promise that resolves to an object matching CountryInfoSchema. + */ +async function getCountryInfo(city: string): Promise> { + try { + const completion = await client.beta.chat.completions.parse({ + model: 'geo-extractor/gpt-4o', + messages: [ + { + role: 'system', + content: 'You are a helpful assistant that provides geographical information.', + }, + { + role: 'user', + // Use a template variable {{input_city}} + content: 'Where is the city {{input_city}} located? Provide the country and continent.', + }, + ], + // Define structured response format using Zod schema + response_format: zodResponseFormat(CountryInfoSchema, "geographical_info"), + // Pass the template variable value via input + // @ts-expect-error input is specific to the WorkflowAI implementation + input: { + input_city: city, // Key matches the template variable name + }, + }); + + // Return the parsed, type-safe object + return completion.choices[0].message.parsed; + + } catch (error) { + console.error('Error fetching or parsing country info:', error); + throw error; + } +} + +/* +// --- Example Usage (Conceptual) --- +async function runGeoExample() { + try { + const parisInfo = await getCountryInfo("Paris"); + console.log("Paris Info:", parisInfo); // Output: { country: 'France', continent: 'Europe' } (example) + + const tokyoInfo = await getCountryInfo("Tokyo"); + console.log("Tokyo Info:", tokyoInfo); // Output: { country: 'Japan', continent: 'Asia' } (example) + } catch (e) { + console.error("Failed to get geo info:", e); + } +} +// runGeoExample(); // Uncomment to run if in an executable context +*/ +``` +Using `input` improves observability in WorkflowAI, as these variables are tracked separately. + +## Enhanced Reliability via Provider Fallback + +WorkflowAI aims for high availability (100% uptime goal) for the core API (`run.workflowai.com`) through redundancy: + +* **AI Provider Fallback (Default: Enabled):** WorkflowAI monitors integrated AI providers (OpenAI, Azure OpenAI, Anthropic, etc.). If your chosen model provider experiences issues, WorkflowAI automatically routes your request to a healthy alternative (e.g., OpenAI API -> Azure OpenAI API) within seconds. This happens seamlessly without code changes. +* **Datacenter Redundancy:** The WorkflowAI API is deployed across multiple geographic regions (e.g., East US, Central US) with automatic traffic redirection via Azure Front Door if a region has problems. + +These features significantly increase your application's resilience. See the [Reliability documentation](/docs/cloud/reliability) for details. + +## Other Features + +The WorkflowAI proxy supports many other advanced features compatible with the `openai` TypeScript SDK: + +* **Streaming:** Use `stream: true` in your `create` call for token-by-token responses. Supported for all models. + ```typescript + async function streamChat() { + const stream = await client.chat.completions.create({ + model: 'streaming-chatbot/gpt-4o-mini', + messages: [{ role: 'user', content: 'Tell me a short story.' }], + stream: true, + }); + for await (const chunk of stream) { + process.stdout.write(chunk.choices[0]?.delta?.content || ''); + } + console.log(); // Newline after stream ends + } + // streamChat(); // Uncomment to run + ``` +* **Cost Calculation:** Access the estimated request cost via `response.cost_usd` (may require accessing the raw response or specific handling depending on the SDK method). +* **Multimodality (Images):** Send image URLs or base64 data in the `messages` array using the standard OpenAI format for models like GPT-4o or Gemini. Combine with structured output for image analysis. +* **Reply-to (Conversations):** Create stateful conversations by passing `reply_to_run_id: "previous_run_id"` in the request. WorkflowAI automatically prepends history. (See main proxy docs). +* **Trace ID (Workflows):** Link multi-step calls into a single workflow trace by passing a consistent `trace_id: "workflow_name/uuid"` in the request for each step. (See main proxy docs). +* **Deployments:** Use server-managed prompt templates and model configurations by specifying `model: "agent-name/#schema_id/deployment_id"`. Allows UI updates without code changes. (See main proxy docs). +* **Tool Calling:** Full support for OpenAI's tool calling feature (`tools`, `tool_choice`, `tool_calls`, `tool` role messages). (See main proxy docs). + +Refer to the main [WorkflowAI OpenAI-Compatible Proxy](/docs/getting-started/proxy) documentation for more detailed explanations and examples of these advanced features. \ No newline at end of file diff --git a/docs/developers/python/openai.md b/docs/developers/python/openai.md new file mode 100644 index 000000000..d5a154113 --- /dev/null +++ b/docs/developers/python/openai.md @@ -0,0 +1,273 @@ +# OpenAI (Python) + +## Basic Usage + +```python +import openai +import os + +# Configure the OpenAI client to use the WorkflowAI endpoint and API key +openai.api_key = os.environ.get("WORKFLOWAI_API_KEY") # Use your WorkflowAI API key +openai.api_base = "https://run.workflowai.com/v1" + +response = openai.ChatCompletion.create( + model="gpt-4", # Or any model supported by your WorkflowAI setup + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello!"} + ] +) + +print(response.choices[0].message.content) +``` + +## Identifying your agent + +```python +response = client.chat.completions.create( + model="my-agent/gpt-4o", + messages=[...] +) + +print(response.choices[0].message.content) +``` + +## Switching models + +```python +# Original call using GPT-4o +response = client.chat.completions.create( + model="my-agent/gpt-4o", + messages=[...] +) + +# Switching to Claude 3.7 Sonnet (verify exact ID on workflowai.com/models) +response = client.chat.completions.create( + model="my-agent/claude-3.7-sonnet-latest", # Simply change the model name + messages=[...] +) + +# Switching to Google's Gemini 2.5 Pro (verify exact ID on workflowai.com/models) +response = client.chat.completions.create( + model="my-agent/gemini/gemini-2.5pro-latest", # Use the appropriate model identifier + messages=[...] +) +``` + +```python +# Assuming 'client' is your configured OpenAI client pointing to WorkflowAI +available_models = client.models.list() + +for model in available_models.data: + print(model.id) + +# This will print the identifiers for all models enabled in your WorkflowAI setup +# e.g., gpt-4o, claude-3-sonnet-latest, gemini-1.5-flash-latest, etc. +``` + +## Reliable Structured Output + +{% hint style="info" %} +**Note on Chatbots:** For purely conversational use cases where you only need text responses and not structured data extraction, this step might not be necessary. +{% endhint %} + +Getting consistent, machine-readable output (like JSON) from language models often requires careful prompt engineering and post-processing. WorkflowAI simplifies this significantly by supporting OpenAI's structured output capabilities, enhanced with broader model compatibility. By defining your desired output structure using Pydantic, you can reliably get parsed data objects without complex prompting or manual validation. + +The `openai` Python library offers a highly convenient way to achieve this by directly providing a [Pydantic](https://docs.pydantic.dev/latest/) model definition. + +To get structured output using the `openai` Python library with WorkflowAI: + +1. Define your desired output structure as a Pydantic `BaseModel`. +2. Use the `client.beta.chat.completions.parse()` method (note the `.parse()` instead of `.create()`). +3. Pass your Pydantic class directly to the `response_format` parameter. +4. Access the parsed Pydantic object directly from `response.choices[0].message.parsed`. + +![Placeholder Structured Output](../assets/proxy/schema.png) + +**Example: Getting Country using a Pydantic Model** + +Let's redefine the `get_country` example using a Pydantic model: + +```python +from pydantic import BaseModel +# Assuming `openai` client is configured as `client` + +class CountryInfo(BaseModel): + country: str + +def get_country(city: str): + # Use the `.parse()` method for structured output with Pydantic + completion = client.beta.chat.completions.parse( + # Use a descriptive agent prefix for organization + model="country-extractor/gpt-4o", + messages=[ + {"role": "system", "content": "You are a helpful assistant that extracts geographical information."}, + {"role": "user", "content": f"What is the country of {city}?"} + ], + # Pass the Pydantic class directly as the response format + response_format=CountryInfo + ) + + parsed_output: CountryInfo = completion.choices[0].message.parsed + return parsed_output +``` + +This approach leverages the `openai` library's integration with Pydantic to abstract away the manual JSON schema definition and response parsing, providing a cleaner developer experience. + +**WorkflowAI Compatibility Advantage:** +A key benefit of using the WorkflowAI proxy is extended compatibility. While native OpenAI structured output requires specific models (like `gpt-4o`), WorkflowAI guarantees this structured output method works **100% of the time across all models** available through the proxy when using the `openai` Python library with Pydantic. You reliably get a parsed object matching your Pydantic class, regardless of the underlying model's native capabilities. + +{% hint style="info" %} +**Team Note:** Clarify in examples that when using Pydantic/structured output, the prompt does *not* need to explicitly ask for JSON output, as WorkflowAI handles the formatting. +{% endhint %} + +## Input Variables + +... + +## Deployments + +```python +# Code after Step 4 +from pydantic import BaseModel +import openai +import os + +# Assuming client is configured for WorkflowAI +client = openai.OpenAI( + api_key=os.environ.get("WORKFLOWAI_API_KEY"), + base_url="https://run.workflowai.com/v1" +) + +class EventDetails(BaseModel): + event_name: str + date: str + time: str + location: str + +email_content = """Subject: Meeting Confirmation + +Hi team, + +Just confirming our project sync meeting for tomorrow, June 15th, at 2:00 PM PST in the Main Conference Room. + +Please come prepared to discuss Q3 planning. + +Thanks, +Alex +""" + +# Call the deployment directly +completion = client.beta.chat.completions.parse( + # Reference agent, schema ID, and deployment ID + model="event-extractor/#1/production", + # messages parameter is no longer needed! + response_format=EventDetails, + extra_body={ + # Only input variable is needed + "input": { + "email_body": email_content + } + } +) + +# Access the parsed Pydantic object as before +parsed_event: EventDetails = completion.choices[0].message.parsed +print(f"Event Name: {parsed_event.event_name}") +print(f"Date: {parsed_event.date}") +# ... and so on +``` + +## Automatic Cost Calculation + +While most standard LLM APIs return usage metrics (like input and output token counts), they typically don't provide the actual monetary cost of the request. Developers are often left to calculate this themselves, requiring them to maintain and apply up-to-date pricing information for each model. + +WorkflowAI simplifies this significantly. The proxy automatically calculates the estimated cost for each LLM request based on the specific model used and WorkflowAI's current pricing data. This calculated cost is then conveniently included directly within the response object returned by the API call, making it easy to access the price for each completion. + +Here's an example of how you access this cost data in Python: + +🚧 [TODO: add tests in autopilot, update backend + code (below) to include latency as well] + +```python +response = client.chat.completions.create( + model="country-extractor/gpt-4o", + messages=[ + {"role": "system", "content": "You extract the country from a given city."}, + {"role": "user", "content": "What country is Paris in?"} + ] +) + +print(f"Estimated cost for this request: ${response.model_extra.get('cost_usd'):.6f}") +# TODO: add latency + +``` + +You can also see the cost in WorkflowAI by going to the Cost page + +![Placeholder Cost](/docs/assets/images/monitoring.png) + +## Multimodality Support + +WorkflowAI extends the OpenAI proxy functionality to support multimodal models, allowing you to process requests that include not only text but also other data types like images, PDFs and other documents, and audio. This is achieved by adhering to the OpenAI API format for multimodal inputs where applicable, or by providing specific WorkflowAI mechanisms. + +### Image Input + +You can send image data directly (as base64 encoded strings) or provide image URLs within the `messages` array, following the standard OpenAI format. WorkflowAI ensures these requests are correctly routed to compatible multimodal models like GPT-4o or Gemini models. + +**Example: Structured Output from Image Analysis** + +Here's how you can use the `openai` Python library with WorkflowAI to get structured data (like the city identified) from an image: + +```python +import openai +import os +from pydantic import BaseModel, Field + +# Configure the OpenAI client to use the WorkflowAI endpoint and API key +client = openai.OpenAI( + api_key=os.environ.get("WORKFLOWAI_API_KEY"), + base_url="https://run.workflowai.com/v1" +) + +# Define the desired structured output using Pydantic +class LocationInfo(BaseModel): + explanation: str = Field(description="Brief explanation of how the city was identified from the image.") + city: str = Field(description="The city depicted in the image.") + +# Use .parse() for structured output +completion = client.beta.chat.completions.parse( + # Use a multimodal model available through WorkflowAI + model="image-analyzer/gpt-4o", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "Analyze this image and identify the city depicted. Provide a brief explanation for your identification."}, + { + "type": "image_url", + "image_url": { + "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/a/a2/Louvre_Courtyard%2C_Looking_West.jpg/2880px-Louvre_Courtyard%2C_Looking_West.jpg", + }, + }, + ], + } + ], + # Specify the Pydantic model as the response format + response_format=LocationInfo +) + +# Access the parsed Pydantic object +location_info: LocationInfo = completion.choices[0].message.parsed +``` + +### PDF Document Input + +{% hint style="info" %} +**Team Note:** Add details and examples for processing PDF documents via the proxy. +{% endhint %} + +### Audio Input + +{% hint style="info" %} +**Team Note:** Add details and examples for processing audio files (e.g., transcription, analysis) via the proxy. +{% endhint %} \ No newline at end of file From e7fdad5d385ea7a69fa08bf0d91029d7776f4f99 Mon Sep 17 00:00:00 2001 From: YBubu Date: Wed, 14 May 2025 15:16:58 +0000 Subject: [PATCH 2/5] fix WOR-4671 --- .../domain/integration_domain/instruction_python_snippets.py | 2 +- .../domain/integration_domain/openai_sdk_python_snippets.py | 2 +- api/core/domain/integration_domain/openai_sdk_ts_snippets.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/core/domain/integration_domain/instruction_python_snippets.py b/api/core/domain/integration_domain/instruction_python_snippets.py index 8558f35fa..6726f861e 100644 --- a/api/core/domain/integration_domain/instruction_python_snippets.py +++ b/api/core/domain/integration_domain/instruction_python_snippets.py @@ -47,6 +47,6 @@ def extract_user_info(user_message: str) -> UserInfo: """ INSTRUCTOR_PYTHON_INTEGRATION_CHAT_AGENT_NAMING_SNIPPET = """response = client.chat.completions.create( - model="[", # Or claude-3-7-sonnet-latest + model="", messages=[{"role": "user", "content": "Hello!"}] )""" diff --git a/api/core/domain/integration_domain/openai_sdk_python_snippets.py b/api/core/domain/integration_domain/openai_sdk_python_snippets.py index db74ce7cb..c7b10f906 100644 --- a/api/core/domain/integration_domain/openai_sdk_python_snippets.py +++ b/api/core/domain/integration_domain/openai_sdk_python_snippets.py @@ -68,6 +68,6 @@ def get_country(city: str) -> CountryInfo: )""" OPENAI_SDK_PYTHON_INTEGRATION_CHAT_AGENT_NAMING_SNIPPET = """response = openai.ChatCompletion.create( - model="[", # e.g. my-agent/gpt-4o-2024-11-20 + model="", messages=[{"role": "user", "content": "Hello!"}] )""" diff --git a/api/core/domain/integration_domain/openai_sdk_ts_snippets.py b/api/core/domain/integration_domain/openai_sdk_ts_snippets.py index 6e4baf1d4..28d2c50e3 100644 --- a/api/core/domain/integration_domain/openai_sdk_ts_snippets.py +++ b/api/core/domain/integration_domain/openai_sdk_ts_snippets.py @@ -86,7 +86,7 @@ # Integration Chat – Suggesting Agent-Prefixed Model Name ---------------------- OPENAI_SDK_TS_INTEGRATION_CHAT_AGENT_NAMING_SNIPPET = """const response = await client.chat.completions.create({ - model: '[', // e.g. my-agent/gpt-4o + model: '', messages: [{ role: 'user', content: 'Hello!' }], }); """ From 3296754eb494a5daac692a8ccd4a8340038ba863 Mon Sep 17 00:00:00 2001 From: YBubu Date: Wed, 14 May 2025 15:39:09 +0000 Subject: [PATCH 3/5] fix test --- api/api/services/internal_tasks/integration_service_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/api/services/internal_tasks/integration_service_test.py b/api/api/services/internal_tasks/integration_service_test.py index 766c086c3..a60164326 100644 --- a/api/api/services/internal_tasks/integration_service_test.py +++ b/api/api/services/internal_tasks/integration_service_test.py @@ -208,7 +208,7 @@ def test_get_agent_naming_code_snippet( assert result.role == "ASSISTANT" assert result.message_kind == MessageKind.agent_naming_code_snippet assert proposed_agent_name in result.content - assert f"[{proposed_agent_name}/" in result.content + assert f"{proposed_agent_name}/" in result.content class TestHasSentAgentNamingCodeSnippet: From 5930787d5f01a0bce6ef12d4338325affe5ce724 Mon Sep 17 00:00:00 2001 From: YBubu Date: Wed, 14 May 2025 16:01:16 +0000 Subject: [PATCH 4/5] updates after MR --- api/api/routers/integrations_router.py | 5 +- .../internal_tasks/integration_service.py | 8 +- .../integration_service_test.py | 4 +- .../internal_tasks/meta_agent_service.py | 175 ++++++++++-------- api/core/agents/integration_agent.py | 2 +- api/core/agents/meta_agent_proxy.py | 69 +++---- .../instruction_python_snippets.py | 0 .../domain/integration/integration_domain.py | 40 ++++ .../integration_domain_test.py | 8 +- .../integration_mapping.py} | 53 +----- .../openai_sdk_python_snippets.py | 0 .../openai_sdk_ts_snippets.py | 0 12 files changed, 192 insertions(+), 172 deletions(-) rename api/core/domain/{integration_domain => integration}/instruction_python_snippets.py (100%) create mode 100644 api/core/domain/integration/integration_domain.py rename api/core/domain/{ => integration}/integration_domain_test.py (95%) rename api/core/domain/{integration_domain/integration_domain.py => integration/integration_mapping.py} (76%) rename api/core/domain/{integration_domain => integration}/openai_sdk_python_snippets.py (100%) rename api/core/domain/{integration_domain => integration}/openai_sdk_ts_snippets.py (100%) diff --git a/api/api/routers/integrations_router.py b/api/api/routers/integrations_router.py index 6c55a90c2..5262bd1f1 100644 --- a/api/api/routers/integrations_router.py +++ b/api/api/routers/integrations_router.py @@ -10,8 +10,9 @@ IntegrationChatMessage, IntegrationChatResponse, ) -from core.domain.integration_domain.integration_domain import OFFICIAL_INTEGRATIONS, IntegrationKind -from core.domain.integration_domain.integration_domain import Integration as DomainIntegration +from core.domain.integration.integration_domain import Integration as DomainIntegration +from core.domain.integration.integration_domain import IntegrationKind +from core.domain.integration.integration_mapping import OFFICIAL_INTEGRATIONS from core.utils.stream_response_utils import safe_streaming_response router = APIRouter(prefix="/v1/integrations") diff --git a/api/api/services/internal_tasks/integration_service.py b/api/api/services/internal_tasks/integration_service.py index eb13b7c66..d7772372f 100644 --- a/api/api/services/internal_tasks/integration_service.py +++ b/api/api/services/internal_tasks/integration_service.py @@ -27,12 +27,14 @@ from core.domain.errors import ObjectNotFoundError from core.domain.events import EventRouter from core.domain.fields.chat_message import ChatMessage -from core.domain.integration_domain.integration_domain import ( +from core.domain.integration.integration_domain import ( + Integration, + IntegrationKind, +) +from core.domain.integration.integration_mapping import ( OFFICIAL_INTEGRATIONS, PROPOSED_AGENT_NAME_AND_MODEL_PLACEHOLDER, WORKFLOWAI_API_KEY_PLACEHOLDER, - Integration, - IntegrationKind, ) from core.domain.task_variant import SerializableTaskVariant from core.domain.users import User diff --git a/api/api/services/internal_tasks/integration_service_test.py b/api/api/services/internal_tasks/integration_service_test.py index a60164326..d4b380f00 100644 --- a/api/api/services/internal_tasks/integration_service_test.py +++ b/api/api/services/internal_tasks/integration_service_test.py @@ -23,11 +23,11 @@ from core.domain.agent_run import AgentRun from core.domain.errors import ObjectNotFoundError from core.domain.events import EventRouter -from core.domain.integration_domain.integration_domain import ( - OFFICIAL_INTEGRATIONS, +from core.domain.integration.integration_domain import ( Integration, IntegrationKind, ) +from core.domain.integration.integration_mapping import OFFICIAL_INTEGRATIONS from core.domain.task_variant import SerializableTaskVariant from core.domain.users import User from core.storage.backend_storage import BackendStorage diff --git a/api/api/services/internal_tasks/meta_agent_service.py b/api/api/services/internal_tasks/meta_agent_service.py index 7bf98e9e9..b71ae29e5 100644 --- a/api/api/services/internal_tasks/meta_agent_service.py +++ b/api/api/services/internal_tasks/meta_agent_service.py @@ -38,12 +38,12 @@ PlaygroundState as PlaygroundStateDomain, ) from core.agents.meta_agent_proxy import ( - GENERIC_PROPOSED_INSTRUCTIONS_INSTRUCTIONS, + GENERIC_INSTRUCTIONS, PROPOSE_INPUT_VARIABLES_INSTRUCTIONS, PROPOSE_NON_OPENAI_MODELS_INSTRUCTIONS, PROPOSE_STRUCTURED_OUTPUT_INSTRUCTIONS, ProxyMetaAgentInput, - proxy_meta_agent_proxy, + proxy_meta_agent, ) from core.agents.meta_agent_proxy import PlaygroundState as ProxyPlaygroundStateDomain from core.agents.meta_agent_proxy import ( @@ -58,10 +58,10 @@ from core.domain.events import EventRouter, MetaAgentChatMessagesSent from core.domain.fields.chat_message import ChatMessage from core.domain.fields.file import File -from core.domain.integration_domain.integration_domain import ( +from core.domain.integration.integration_domain import ( ProgrammingLanguage, - default_integration_for_language, ) +from core.domain.integration.integration_mapping import default_integration_for_language from core.domain.models.model_data import LatestModel from core.domain.models.model_datas_mapping import MODEL_DATAS from core.domain.models.model_provider_datas_mapping import AZURE_PROVIDER_DATA, OPENAI_PROVIDER_DATA @@ -280,6 +280,11 @@ def to_domain(self) -> GenerateAgentInputToolCallResult: class MetaAgentChatMessage(BaseModel): + # TODO: make this non nullable + sent_at: datetime.datetime | None = Field( + default=None, + description="The date and time the message was sent", + ) role: Literal["USER", "PLAYGROUND", "ASSISTANT"] = Field( description="The role of the message sender, 'USER' is the actual human user browsing the playground, 'PLAYGROUND' are automated messages sent by the playground to the agent, and 'ASSISTANT' being the assistant generated by the agent", ) @@ -881,8 +886,9 @@ async def stream_meta_agent_response( messages: list[MetaAgentChatMessage], playground_state: PlaygroundState, ) -> AsyncIterator[list[MetaAgentChatMessage]]: + now = datetime.datetime.now() if len(messages) == 0: - yield [MetaAgentChatMessage(role="ASSISTANT", content=FIRST_MESSAGE_CONTENT)] + yield [MetaAgentChatMessage(role="ASSISTANT", content=FIRST_MESSAGE_CONTENT, sent_at=now)] return current_agent = await self.storage.task_variant_latest_by_schema_id(task_tuple[0], agent_schema_id) @@ -907,6 +913,7 @@ async def stream_meta_agent_response( role="ASSISTANT", content=chunk.output.content, feedback_token=chunk.feedback_token, + sent_at=now, ), ] yield ret @@ -920,6 +927,7 @@ async def stream_meta_agent_response( content=assistant_message_content, tool_call=tool_call, feedback_token=chunk.feedback_token, + sent_at=now, ), ] yield ret @@ -1093,7 +1101,7 @@ async def _build_proxy_meta_agent_input( ), workflowai_documentation_sections=await DocumentationService().get_relevant_doc_sections( chat_messages=[message.to_chat_message() for message in messages], - agent_instructions=GENERIC_PROPOSED_INSTRUCTIONS_INSTRUCTIONS or "", + agent_instructions=GENERIC_INSTRUCTIONS or "", ), integration_documentation=[], # Will be filled in later available_tools_description=internal_tools_description( @@ -1198,6 +1206,16 @@ def _is_only_using_openai_models(self, agent_runs: list[AgentRun]) -> bool: return True + def _is_message_kind_already_sent( + self, + messages: list[MetaAgentChatMessage], + message_kind: MetaAgentChatMessageKind, + ) -> bool: + for message in messages: + if message.kind == message_kind: + return True + return False + async def stream_proxy_meta_agent_response( self, task_tuple: TaskTuple, @@ -1205,8 +1223,9 @@ async def stream_proxy_meta_agent_response( user_email: str | None, messages: list[MetaAgentChatMessage], ) -> AsyncIterator[list[MetaAgentChatMessage]]: + now = datetime.datetime.now() if len(messages) == 0: - yield [MetaAgentChatMessage(role="ASSISTANT", content=FIRST_MESSAGE_CONTENT)] + yield [MetaAgentChatMessage(role="ASSISTANT", content=FIRST_MESSAGE_CONTENT, sent_at=now)] return current_agent = await self.storage.task_variant_latest_by_schema_id(task_tuple[0], agent_schema_id) @@ -1239,80 +1258,84 @@ async def stream_proxy_meta_agent_response( is_user_triggered = self._is_user_triggered(messages) and messages[-1].content not in ["POLL", "poll"] is_using_instruction_variables = current_agent.input_schema.json_schema.get("properties", None) is not None is_using_structured_generation = not current_agent.output_schema.json_schema.get("format") == "message" + has_tried_other_models = not self._is_only_using_openai_models(agent_runs) + + message_kind = "non_specific" + if not is_user_triggered: + if ( + agent_runs + and not has_tried_other_models + and not self._is_message_kind_already_sent(messages, "try_other_models_proposal") + ): + # "Try non-OpenAI model" use case + instructions = PROPOSE_NON_OPENAI_MODELS_INSTRUCTIONS + message_kind = "try_other_models_proposal" + elif ( + agent_runs + and has_tried_other_models + and not is_using_instruction_variables + and not self._is_message_kind_already_sent(messages, "setup_input_variables_proposal") + ): + # "Migrate to input variables" use case + input_variables_extractor_agent_run = await input_variables_extractor_agent( + InputVariablesExtractorInput( + agent_inputs=[agent_run.task_input for agent_run in agent_runs], + ), + ) + proxy_meta_agent_input.suggested_instructions_with_input_variables = ( + input_variables_extractor_agent_run.instructions_with_input_variables + ) + proxy_meta_agent_input.suggested_input_variables_example = ( + input_variables_extractor_agent_run.input_variables_example + ) + instructions = PROPOSE_INPUT_VARIABLES_INSTRUCTIONS + message_kind = "setup_input_variables_proposal" + elif ( + agent_runs + and has_tried_other_models + and is_using_instruction_variables # We require the user to have used input variables before proposing structured output + and not is_using_structured_generation + and not self._is_message_kind_already_sent(messages, "setup_structured_output_proposal") + ): + # "Migrate to structured output" use case + output_schema_extractor_run = await output_schema_extractor_agent( + OutputSchemaExtractorInput( + agent_runs=[ + OutputSchemaExtractorInput.AgentRun( + raw_messages=[ + llm_completion.messages or [] for llm_completion in agent_run.llm_completions or [] + ], + input=str(agent_run.task_input), + output=str(agent_run.task_output), + ) + for agent_run in agent_runs + ], + programming_language=integration.programming_language, + structured_object_class=integration.output_class, + ), + ) + proxy_meta_agent_input.suggested_output_class_code = ( + output_schema_extractor_run.proposed_structured_object_class + ) + proxy_meta_agent_input.suggested_instructions_parts_to_remove = ( + output_schema_extractor_run.instructions_parts_to_remove + ) + instructions = PROPOSE_STRUCTURED_OUTPUT_INSTRUCTIONS + message_kind = "setup_structured_output_proposal" + else: + # This is a polling without required action, return. + yield [] + return - if not is_user_triggered and agent_runs and self._is_only_using_openai_models(agent_runs): - # "Try non-OpenAI model" use case - - instructions = PROPOSE_NON_OPENAI_MODELS_INSTRUCTIONS - elif not is_user_triggered and agent_runs and not is_using_instruction_variables: - # "Migrate to input variables" use case - input_variables_extractor_agent_run = await input_variables_extractor_agent( - InputVariablesExtractorInput( - agent_inputs=[agent_run.task_input for agent_run in agent_runs], - ), - ) - proxy_meta_agent_input.suggested_instructions_with_input_variables = ( - input_variables_extractor_agent_run.instructions_with_input_variables - ) - proxy_meta_agent_input.suggested_input_variables_example = ( - input_variables_extractor_agent_run.input_variables_example - ) - instructions = PROPOSE_INPUT_VARIABLES_INSTRUCTIONS - elif not is_user_triggered and agent_runs and not is_using_structured_generation: - # "Migrate to structured output" use case - - output_schema_extractor_run = await output_schema_extractor_agent( - OutputSchemaExtractorInput( - agent_runs=[ - OutputSchemaExtractorInput.AgentRun( - raw_messages=[ - llm_completion.messages or [] for llm_completion in agent_run.llm_completions or [] - ], - input=str(agent_run.task_input), - output=str(agent_run.task_output), - ) - for agent_run in agent_runs - ], - programming_language=integration.programming_language, - structured_object_class=integration.output_class, - ), - ) - proxy_meta_agent_input.suggested_output_class_code = ( - output_schema_extractor_run.proposed_structured_object_class - ) - proxy_meta_agent_input.suggested_instructions_parts_to_remove = ( - output_schema_extractor_run.instructions_parts_to_remove - ) - instructions = PROPOSE_STRUCTURED_OUTPUT_INSTRUCTIONS - # All other use cases else: - instructions = GENERIC_PROPOSED_INSTRUCTIONS_INSTRUCTIONS + # This is an actual client message. Answer + instructions = GENERIC_INSTRUCTIONS + message_kind = "non_specific" ret: list[MetaAgentChatMessage] = [] - """ - chunk: "workflowai.Run[ProxyMetaAgentOutput] | None" = None - async for chunk in proxy_meta_agent.stream( - proxy_meta_agent_input, - version=workflowai.VersionProperties( - model=model, - instructions=instructions, - temperature=0.5, - ), - ): - if chunk.output.assistant_answer: - ret = [ - MetaAgentChatMessage( - role="ASSISTANT", - content=chunk.output.assistant_answer.replace("REPLACE_INSTRUCTIONS", instructions_to_inject), - feedback_token=chunk.feedback_token, - ), - ] - yield ret - """ - accumulator = "" - async for chunk in proxy_meta_agent_proxy( + async for chunk in proxy_meta_agent( proxy_meta_agent_input, instructions, ): @@ -1322,6 +1345,8 @@ async def stream_proxy_meta_agent_response( MetaAgentChatMessage( role="ASSISTANT", content=accumulator, + sent_at=now, + kind=message_kind, ), ] yield ret diff --git a/api/core/agents/integration_agent.py b/api/core/agents/integration_agent.py index 552ce32ba..0011c00ce 100644 --- a/api/core/agents/integration_agent.py +++ b/api/core/agents/integration_agent.py @@ -5,7 +5,7 @@ from pydantic import BaseModel, Field from core.domain.documentation_section import DocumentationSection -from core.domain.integration_domain.integration_domain import Integration +from core.domain.integration.integration_domain import Integration class IntegrationAgentChatMessage(BaseModel): diff --git a/api/core/agents/meta_agent_proxy.py b/api/core/agents/meta_agent_proxy.py index 204d4ecfc..70ec536dc 100644 --- a/api/core/agents/meta_agent_proxy.py +++ b/api/core/agents/meta_agent_proxy.py @@ -1,13 +1,12 @@ import datetime from typing import Any, AsyncIterator, Literal, Self -import workflowai from openai import AsyncOpenAI from pydantic import BaseModel, Field from core.domain.documentation_section import DocumentationSection from core.domain.feedback import Feedback -from core.domain.integration_domain.integration_domain import Integration +from core.domain.integration.integration_domain import Integration from core.domain.url_content import URLContent from core.domain.version_environment import VersionEnvironment @@ -512,46 +511,19 @@ class ProxyMetaAgentOutput(BaseModel): Always refer to the 'integration_documentation' that matchers 'playground_state.current_agent.used_integration' for code block language and syntax. """ -GENERIC_PROPOSED_INSTRUCTIONS_INSTRUCTIONS = f""" -{_PROXY_META_AGENT_COMMON_INSTRUCTIONS} - -# 1. Trying non-OpenAI models -- Check if the user is using OpenAI models, for example GPT-4, GPT-4 Mini, O1, O3, O4, etc. And if so, you can suggest they try other models from providers Claude, Llama, Grok. You can check all models in the 'available_models' section of the playground state. +_PROPOSE_NON_OPENAI_MODELS_INSTRUCTIONS = """ +Your promise goal here is to make the user migrate to non-OpenAI models. You can check the documentation to see why it's a good idea to use non-OpenAI models. The reason to use different models than OpenAI is having better performing models, cheaper models, faster models. -If you detect that the user is still using OpenAI models, you can suggest them to switch try different models by first trying the models in the Playground and then modifying the model string in their code (as explained in the instruction documentation). - -# 2. Suggest to use input variables - -As mentioned in the WorkflowAI documentation, using input variables has a lot of advantages when possible. It allows for observability and other features, and also opens the door for deployments. - -## How to spot if the user is using input variables or not ? - -You can know that the user is using input variables if you see variables names surrounded by double curly braces. -If using input variables, the actual variable values will need to be passed in the 'input' payload of the 'raw_run_request'. So, you will need to double-check that as well because if the user omits passing variable values in the 'raw_run_request.input', the agent will not work as expected. - -So if the user is not using yet input variables or badly, you need to suggest him to make the update. Inform your argumentation based on what you will find in the documentation. Note that the exact way of injecting input variables depends on the integration used, but is often by passing 'extra_body'...'. - -Please be aware that your suggestions are optional for the user and if the user declines to use a non-OpenAI model, don't bother them with that again. You can suggest them the next step which is using the input variables. And so on with the structured generation. - -You must check that the user is leveraging WorkflowAI's feature for his agent in the order below (1., 2., 3.). After checking each criteria, you can go to the next one if the user is already using this feature. Do not mention the steps that are solved already. -""" +You must pick a relevant model for the use case of the client and show him some code blocks that show how to update the model.""" PROPOSE_NON_OPENAI_MODELS_INSTRUCTIONS = f""" {_PROXY_META_AGENT_COMMON_INSTRUCTIONS} - -Your precise goal here is to make the user migrate to non-OpenAI models. You can check the documentation to see why it's a good idea to use non-OpenAI models. - -The reason to use different models than OpenAI is having better performing models, cheaper models, faster models. - -You must pick a relevant model for the use case of the client and show him some code blocks that show how to update the model. +{_PROPOSE_NON_OPENAI_MODELS_INSTRUCTIONS} """ -PROPOSE_INPUT_VARIABLES_INSTRUCTIONS = f""" -{_PROXY_META_AGENT_COMMON_INSTRUCTIONS} - -Your precise goal here is to make the user migrate to input variables. You can check the documentation to see why it's a good idea to use input variables. +_PROPOSE_INPUT_VARIABLES_INSTRUCTIONS = """Your promise goal here is to make the user migrate to input variables. You can check the documentation to see why it's a good idea to use input variables. Use the 'suggested_instructions_with_input_variables' and 'suggested_input_variables_example' in order to provide code snippet that are valid in the context of the WorkflowAI integrations documented in the 'workflowai_documentation_sections' section of the input.` Start by 1. Explaining to the user why it's a good idea to use input variables. Start with a simple phrase like "I have something to propose to make your to unlock a lot of WorkflowAI's capabilities: ..." @@ -560,9 +532,11 @@ class ProxyMetaAgentOutput(BaseModel): When instructions are spread over several messages, make sure to display a code block that showed several messages with the right instructions at the right place. """ -PROPOSE_STRUCTURED_OUTPUT_INSTRUCTIONS = f""" -{_PROXY_META_AGENT_COMMON_INSTRUCTIONS} +PROPOSE_INPUT_VARIABLES_INSTRUCTIONS = f""" +{_PROPOSE_INPUT_VARIABLES_INSTRUCTIONS} +""" +_PROPOSE_STRUCTURED_OUTPUT_INSTRUCTIONS = """ Your precise goal here is to make the user migrate to structured output. You can check the documentation to see why it's a good idea to use structured output. Use the 'suggested_output_class_code' and 'suggested_instructions_parts_to_remove' in order to provide code snippet that are valid in the context of the WorkflowAI integrations documented in the 'workflowai_documentation_sections' section of the input.` If 'suggested_instructions_parts_to_remove' are fed, you can mention to the user that they can remove those parts in their existing instructions that talk about generating a valid JSON, because this is not needed anymore when you use structure generation. @@ -573,17 +547,28 @@ class ProxyMetaAgentOutput(BaseModel): If you mention a SDK or package, etc., make sure you are mentioning the right one, for example "Instructor". You do not need to rebuild the full code snippets, just higlight the main changes to do and a few lines of code around it. -Be aware that the user can just update his code and has nothing to do in the interface. Just updating the code is enough, and a new schema and the agent will be automatically updated in WorkflowAI.com. +Be aware that the user can just update his code and has nothing to do in the interface. Just updating the code is enough, and a new schema and the agent will be automatically updated in WorkflowAI.com.""" + +PROPOSE_STRUCTURED_OUTPUT_INSTRUCTIONS = f""" +{_PROXY_META_AGENT_COMMON_INSTRUCTIONS} +{_PROPOSE_STRUCTURED_OUTPUT_INSTRUCTIONS} """ +GENERIC_INSTRUCTIONS = f""" +{_PROXY_META_AGENT_COMMON_INSTRUCTIONS} -@workflowai.agent( - id="proxy-meta-agent", -) -async def proxy_meta_agent(_: ProxyMetaAgentInput) -> ProxyMetaAgentOutput: ... +# In case the user enquires a about testing new models: +{_PROPOSE_NON_OPENAI_MODELS_INSTRUCTIONS} + +# In case the user enquires a about input variables: +{_PROPOSE_INPUT_VARIABLES_INSTRUCTIONS} + +# In case the user enquires a about structured output: +{_PROPOSE_STRUCTURED_OUTPUT_INSTRUCTIONS} +""" -async def proxy_meta_agent_proxy(input: ProxyMetaAgentInput, instructions: str) -> AsyncIterator[ProxyMetaAgentOutput]: +async def proxy_meta_agent(input: ProxyMetaAgentInput, instructions: str) -> AsyncIterator[ProxyMetaAgentOutput]: client = AsyncOpenAI( api_key="wai-4hcqxDO3eZLytkIsLdSHGLbviAP0P16bRoX6AVGLTFM", base_url="https://run.workflowai.dev/v1", diff --git a/api/core/domain/integration_domain/instruction_python_snippets.py b/api/core/domain/integration/instruction_python_snippets.py similarity index 100% rename from api/core/domain/integration_domain/instruction_python_snippets.py rename to api/core/domain/integration/instruction_python_snippets.py diff --git a/api/core/domain/integration/integration_domain.py b/api/core/domain/integration/integration_domain.py new file mode 100644 index 000000000..059af59ad --- /dev/null +++ b/api/core/domain/integration/integration_domain.py @@ -0,0 +1,40 @@ +from enum import Enum + +from pydantic import BaseModel + + +class IntegrationPartner(str, Enum): + INSTRUCTOR = "instructor" + OPENAI_SDK = "openai-sdk" + OPENAI_SDK_TS = "openai-sdk-ts" + + +class ProgrammingLanguage(str, Enum): + PYTHON = "python" + TYPESCRIPT = "typescript" + + +class IntegrationKind(str, Enum): + INSTRUCTOR_PYTHON = "instructor-python" + OPENAI_SDK_PYTHON = "openai-sdk-python" + OPENAI_SDK_TS = "openai-sdk-ts" + + +class Integration(BaseModel): + integration_partner: IntegrationPartner + programming_language: ProgrammingLanguage + default_for_language: bool + output_class: str + display_name: str + slug: IntegrationKind + + logo_url: str + landing_page_snippet: str + + # Not all integration may support structured generation. + landing_page_structured_generation_snippet: str | None = None + + integration_chat_initial_snippet: str + integration_chat_agent_naming_snippet: str + + documentation_filepaths: list[str] diff --git a/api/core/domain/integration_domain_test.py b/api/core/domain/integration/integration_domain_test.py similarity index 95% rename from api/core/domain/integration_domain_test.py rename to api/core/domain/integration/integration_domain_test.py index 7a5a2115e..4a69431d3 100644 --- a/api/core/domain/integration_domain_test.py +++ b/api/core/domain/integration/integration_domain_test.py @@ -1,9 +1,11 @@ -from core.domain.integration_domain.integration_domain import ( +from core.domain.integration.integration_domain import ( + IntegrationKind, + ProgrammingLanguage, +) +from core.domain.integration.integration_mapping import ( OFFICIAL_INTEGRATIONS, PROPOSED_AGENT_NAME_AND_MODEL_PLACEHOLDER, WORKFLOWAI_API_KEY_PLACEHOLDER, - IntegrationKind, - ProgrammingLanguage, ) diff --git a/api/core/domain/integration_domain/integration_domain.py b/api/core/domain/integration/integration_mapping.py similarity index 76% rename from api/core/domain/integration_domain/integration_domain.py rename to api/core/domain/integration/integration_mapping.py index d58577216..f83b2a3fb 100644 --- a/api/core/domain/integration_domain/integration_domain.py +++ b/api/core/domain/integration/integration_mapping.py @@ -1,20 +1,22 @@ -from enum import Enum - -from pydantic import BaseModel - -from core.domain.integration_domain.instruction_python_snippets import ( +from core.domain.integration.instruction_python_snippets import ( INSTRUCTOR_PYTHON_INTEGRATION_CHAT_AGENT_NAMING_SNIPPET, INSTRUCTOR_PYTHON_INTEGRATION_CHAT_INITIAL_SNIPPET, INSTRUCTOR_PYTHON_LANDING_PAGE_SNIPPET, INSTRUCTOR_PYTHON_LANDING_PAGE_STRUCTURED_GENERATION_SNIPPET, ) -from core.domain.integration_domain.openai_sdk_python_snippets import ( +from core.domain.integration.integration_domain import ( + Integration, + IntegrationKind, + IntegrationPartner, + ProgrammingLanguage, +) +from core.domain.integration.openai_sdk_python_snippets import ( OPENAI_SDK_PYTHON_INTEGRATION_CHAT_AGENT_NAMING_SNIPPET, OPENAI_SDK_PYTHON_INTEGRATION_CHAT_INITIAL_SNIPPET, OPENAI_SDK_PYTHON_LANDING_PAGE_SNIPPET, OPENAI_SDK_PYTHON_LANDING_PAGE_STRUCTURED_GENERATION_SNIPPET, ) -from core.domain.integration_domain.openai_sdk_ts_snippets import ( +from core.domain.integration.openai_sdk_ts_snippets import ( OPENAI_SDK_TS_INTEGRATION_CHAT_AGENT_NAMING_SNIPPET, OPENAI_SDK_TS_INTEGRATION_CHAT_INITIAL_SNIPPET, OPENAI_SDK_TS_LANDING_PAGE_SNIPPET, @@ -25,43 +27,6 @@ PROPOSED_AGENT_NAME_AND_MODEL_PLACEHOLDER = "" -class IntegrationPartner(str, Enum): - INSTRUCTOR = "instructor" - OPENAI_SDK = "openai-sdk" - OPENAI_SDK_TS = "openai-sdk-ts" - - -class ProgrammingLanguage(str, Enum): - PYTHON = "python" - TYPESCRIPT = "typescript" - - -class IntegrationKind(str, Enum): - INSTRUCTOR_PYTHON = "instructor-python" - OPENAI_SDK_PYTHON = "openai-sdk-python" - OPENAI_SDK_TS = "openai-sdk-ts" - - -class Integration(BaseModel): - integration_partner: IntegrationPartner - programming_language: ProgrammingLanguage - default_for_language: bool - output_class: str - display_name: str - slug: IntegrationKind - - logo_url: str - landing_page_snippet: str - - # Not all integration may support structured generation. - landing_page_structured_generation_snippet: str | None = None - - integration_chat_initial_snippet: str - integration_chat_agent_naming_snippet: str - - documentation_filepaths: list[str] - - OFFICIAL_INTEGRATIONS = [ Integration( integration_partner=IntegrationPartner.INSTRUCTOR, diff --git a/api/core/domain/integration_domain/openai_sdk_python_snippets.py b/api/core/domain/integration/openai_sdk_python_snippets.py similarity index 100% rename from api/core/domain/integration_domain/openai_sdk_python_snippets.py rename to api/core/domain/integration/openai_sdk_python_snippets.py diff --git a/api/core/domain/integration_domain/openai_sdk_ts_snippets.py b/api/core/domain/integration/openai_sdk_ts_snippets.py similarity index 100% rename from api/core/domain/integration_domain/openai_sdk_ts_snippets.py rename to api/core/domain/integration/openai_sdk_ts_snippets.py From 8e87631e28e8bd2d008da78e0a55c54faa74016e Mon Sep 17 00:00:00 2001 From: YBubu Date: Wed, 14 May 2025 16:16:09 +0000 Subject: [PATCH 5/5] fix test --- .../internal_tasks/meta_agent_service_test.py | 44 ++++++++++++++++--- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/api/api/services/internal_tasks/meta_agent_service_test.py b/api/api/services/internal_tasks/meta_agent_service_test.py index 5b1d81870..c36c7fe73 100644 --- a/api/api/services/internal_tasks/meta_agent_service_test.py +++ b/api/api/services/internal_tasks/meta_agent_service_test.py @@ -4,6 +4,7 @@ from unittest.mock import AsyncMock, Mock, patch import pytest +from freezegun import freeze_time from pydantic import BaseModel from api.services.documentation_service import DocumentationService @@ -237,12 +238,20 @@ async def test_build_meta_agent_input( assert product.name == expected_input.company_context.company_products[i].name assert product.description == expected_input.company_context.company_products[i].description + # Freeze the "now" + @freeze_time("2025-04-17T12:56:41.413541") @pytest.mark.parametrize( "user_email, messages, meta_agent_chunks, expected_outputs", [ ( "user@example.com", - [MetaAgentChatMessage(role="USER", content="Hello")], + [ + MetaAgentChatMessage( + sent_at=datetime.datetime(2025, 4, 17, 12, 56, 41, 413541), + role="USER", + content="Hello", + ), + ], [ MetaAgentOutput( content="Hi there!", @@ -252,19 +261,43 @@ async def test_build_meta_agent_input( ), ], [ - [MetaAgentChatMessage(role="ASSISTANT", content="Hi there!")], - [MetaAgentChatMessage(role="ASSISTANT", content="How can I help you today?")], + [ + MetaAgentChatMessage( + sent_at=datetime.datetime(2025, 4, 17, 12, 56, 41, 413541), + role="ASSISTANT", + content="Hi there!", + ), + ], + [ + MetaAgentChatMessage( + sent_at=datetime.datetime(2025, 4, 17, 12, 56, 41, 413541), + role="ASSISTANT", + content="How can I help you today?", + ), + ], ], ), ( None, - [MetaAgentChatMessage(role="USER", content="Help")], + [ + MetaAgentChatMessage( + sent_at=datetime.datetime(2025, 4, 17, 12, 56, 41, 413541), + role="USER", + content="Help", + ), + ], [ MetaAgentOutput(content=None), # Empty chunk MetaAgentOutput(content="I can help with WorkflowAI!"), ], [ - [MetaAgentChatMessage(role="ASSISTANT", content="I can help with WorkflowAI!")], + [ + MetaAgentChatMessage( + sent_at=datetime.datetime(2025, 4, 17, 12, 56, 41, 413541), + role="ASSISTANT", + content="I can help with WorkflowAI!", + ), + ], ], ), ( @@ -274,6 +307,7 @@ async def test_build_meta_agent_input( [ [ MetaAgentChatMessage( + sent_at=datetime.datetime(2025, 4, 17, 12, 56, 41, 413541), role="ASSISTANT", content="Hi, I'm WorkflowAI's agent. How can I help you?", ),