diff --git a/PULL_REQUEST.md b/PULL_REQUEST.md new file mode 100644 index 0000000..24f6b93 --- /dev/null +++ b/PULL_REQUEST.md @@ -0,0 +1,105 @@ +# Add Support for Qwen3-Thinking Models via Function Calling + +## Overview + +This PR introduces comprehensive support for **qwen3-thinking models**, which do not support Structured Output (SO) but work excellently through Function Calling (FC). The implementation maintains full compatibility with the SGR framework while enabling the use of thinking models that provide intermediate reasoning in their outputs. + +## Key Features + +### 1. Universal Pydantic to Function Calling Converter +- **File**: `sgr_deep_research/core/utils/pydantic_convert.py` +- Automatic JSON Schema generation from Pydantic models +- Support for complex types: `Literal`, `Optional`, `Union`, `List`, `Dict` +- Handles nested models and constraints (min/max, length, pattern) + +### 2. Qwen3-Thinking Response Adapter +- **File**: `sgr_deep_research/core/adapters/qwen3_thinking_adapter.py` +- Three-strategy extraction from "dirty" thinking model outputs: + - Strategy 1: `tool_calls` with JSON in `arguments` + - Strategy 2: `content` with `...` tags (priority) + - Strategy 3: `content` with raw JSON (fallback) +- Robust parsing with detailed error diagnostics + +### 3. SGRQwen3ThinkingAgent +- **File**: `sgr_deep_research/core/agents/sgr_qwen3_thinking_agent.py` +- Full-featured SGR agent adapted for thinking models +- Uses Function Calling instead of Structured Output +- Modified system prompt with thinking model instructions +- Maintains complete SGR architecture: reasoning → action → evaluation + +## Documentation & Examples + +- **Comprehensive Documentation**: `docs/QWEN3_THINKING_SUPPORT.md` + - Detailed component descriptions + - Configuration examples + - Usage patterns + - Troubleshooting guide + - Performance comparison with instruct models + +- **Practical Examples**: `examples/qwen3_thinking_example.py` + - Basic usage + - Clarification handling + - Configuration loading + +## ️ Architecture + +``` +SGRQwen3ThinkingAgent +│ +├── Pydantic → FC Conversion (pydantic_convert.py) +│ └── Auto-generates OpenAI Function Calling schema +│ +├── Modified System Prompt +│ └── Includes thinking model instructions + dynamic schema +│ +├── Reasoning Phase (Function Calling) +│ └── LLM generates reasoning + tool call +│ +├── Response Extraction (qwen3_thinking_adapter.py) +│ └── Extracts structured data from mixed output +│ +└── Action & Evaluation + └── Standard SGR flow +``` + +## Design Decisions + +1. **Function Calling over Structured Output**: Thinking models don't support SO, but FC provides equivalent functionality while preserving reasoning visibility + +2. **Multi-Strategy Extraction**: Thinking models can output in various formats depending on vLLM configuration - the adapter handles all cases gracefully + +3. **Modified System Prompt**: Incorporates base prompt from config + schema + thinking-specific instructions + +4. **Full SGR Compatibility**: Maintains all SGR agent features (clarifications, planning, tool selection, etc.) + +## Backward Compatibility + +This PR: +- Does not modify existing agents or tools +- Adds new optional components +- Works alongside standard SGR agents +- Uses existing configuration system from `agents-config-definitions` branch + +## Configuration + +### config.yaml +```yaml +llm: + model: "Qwen/Qwen3-14B-thinking" + temperature: 0.3 + max_tokens: 12000 +``` + +### agents.yaml +```yaml +agents: + - name: "qwen3_thinking_agent" + base_class: "SGRQwen3ThinkingAgent" + llm: + model: "Qwen/Qwen3-14B-thinking" + tools: + - "WebSearchTool" + - "CreateReportTool" + - "FinalAnswerTool" +``` + diff --git a/README.md b/README.md index 5ea4cf0..b35b124 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,6 @@ Recent benchmarks and validation experiments were conducted in collaboration wit Learn more about the company: [redmadrobot.ai](https://redmadrobot.ai/) ↗️ - ## Star History [![Star History Chart](https://api.star-history.com/svg?repos=vamplabAI/sgr-deep-research&type=Date)](https://star-history.com/#vamplabAI/sgr-deep-research&Date) diff --git a/agents.yaml.example b/agents.yaml.example new file mode 100644 index 0000000..133798f --- /dev/null +++ b/agents.yaml.example @@ -0,0 +1,143 @@ +# Example Custom Agents Configuration +# ===================================== +# This file demonstrates how to define custom agents for SGR Deep Research + +# Notes: +# ------ +# 1. Agent names must be unique or will be overridden +# 2. All tools must be registered in the tool registry +# 3. LLM, Search, Prompts, Execution, MCP settings are optional and inherit from global config +# 4. Agents override global settings by providing their own values + +agents: + # Example 1: Simple custom research agent with overrides + - name: "custom_research_agent" + base_class: "SGRResearchAgent" + + # Optional: Override LLM settings for this agent + llm: + model: "gpt-4o" + temperature: 0.3 + max_tokens: 16000 + # api_key: "your-custom-api-key" # Optional: use different API key + # base_url: "https://api.openai.com/v1" # Optional: use different endpoint + # proxy: "http://127.0.0.1:8080" # Optional: use proxy + + # Optional: Override search settings + search: + max_results: 15 + max_pages: 8 + content_limit: 2000 + + # Optional: Custom prompts for this agent + prompts: + # Option 1: Use custom prompt files + system_prompt_file: "prompts/custom_system_prompt.txt" + initial_user_request_file: "prompts/custom_initial_request.txt" + clarification_response_file: "prompts/custom_clarification.txt" + + # Option 2: Provide prompts directly (uncomment to use) + # system_prompt_str: "You are a specialized research agent..." + # initial_user_request_str: "Custom initial request template..." + # clarification_response_str: "Custom clarification template..." + + # Optional: Execution configuration + execution: + max_steps: 8 + max_iterations: 15 + max_clarifications: 5 + max_searches: 6 + mcp_context_limit: 20000 + logs_dir: "logs/custom_agent" + reports_dir: "reports/custom_agent" + + # Optional: MCP configuration + mcp: + mcpServers: + deepwiki: + url: "https://mcp.deepwiki.com/mcp" + + # Tools this agent can use (must be registered in tool registry) + tools: + - "WebSearchTool" + - "ExtractPageContentTool" + - "CreateReportTool" + - "ClarificationTool" + - "GeneratePlanTool" + - "AdaptPlanTool" + - "FinalAnswerTool" + + # Example 2: Minimal agent with defaults + - name: "simple_agent" + base_class: "SGRToolCallingResearchAgent" + + # Only override what's needed + llm: + model: "gpt-4o-mini" + + tools: + - "WebSearchTool" + - "FinalAnswerTool" + + # Example 3: Fast research agent optimized for speed + - name: "fast_research_agent" + base_class: "SGRToolCallingResearchAgent" + + llm: + model: "gpt-4o-mini" + temperature: 0.1 + max_tokens: 4000 + + execution: + max_steps: 4 + max_iterations: 8 + max_clarifications: 2 + max_searches: 3 + + tools: + - "WebSearchTool" + - "CreateReportTool" + - "FinalAnswerTool" + - "ReasoningTool" + + # Example 4: Specialized technical analyst with custom prompts + - name: "technical_analyst" + base_class: "SGRResearchAgent" + + llm: + model: "gpt-4o" + temperature: 0.2 + + prompts: + system_prompt_file: "prompts/technical_analyst_prompt.txt" + + execution: + max_steps: 10 + max_iterations: 20 + max_clarifications: 3 + max_searches: 8 + + tools: + - "WebSearchTool" + - "ExtractPageContentTool" + - "CreateReportTool" + - "ClarificationTool" + - "FinalAnswerTool" + + # Example 5: Agent using inline prompts instead of files + - name: "inline_prompt_agent" + base_class: "SGRResearchAgent" + + prompts: + system_prompt_str: | + You are a helpful research assistant. + Your goal is to provide accurate and concise information. + initial_user_request_str: | + User request: {user_request} + Please analyze and respond. + clarification_response_str: | + I need clarification on: {clarification_needed} + + tools: + - "WebSearchTool" + - "FinalAnswerTool" diff --git a/benchmark/run_simpleqa_bench.py b/benchmark/run_simpleqa_bench.py index cb4c415..ed92c88 100644 --- a/benchmark/run_simpleqa_bench.py +++ b/benchmark/run_simpleqa_bench.py @@ -14,7 +14,7 @@ grading_answer, save_result, ) -from sgr_deep_research.settings import get_config +from sgr_deep_research.core.agent_config import GlobalConfig project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) config_path = os.path.join(project_root, "config.yaml") @@ -29,8 +29,8 @@ async def benchmark_agent(question, answer, model_config) -> Dict[str, Any]: - system_conf = get_config() - agent = BenchmarkAgent(task=question, max_iterations=system_conf.execution.max_steps) + system_conf = GlobalConfig() + agent = BenchmarkAgent(task=question, max_iterations=system_conf.execution.max_iterations) try: await agent.execute() diff --git a/config.yaml.example b/config.yaml.example index c619095..03f1a09 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -1,58 +1,55 @@ -# SGR Research Agent - Configuration Template -# Production-ready configuration for Schema-Guided Reasoning -# Copy this file to config.yaml and fill in your API keys - -# OpenAI API Configuration -openai: - api_key: "your-openai-api-key-here" # Required: Your OpenAI API key - base_url: "https://api.openai.com/v1" # Optional: Alternative URL (e.g., for proxy LiteLLM/vLLM) - model: "gpt-4o-mini" # Model to use - max_tokens: 8000 # Maximum number of tokens - temperature: 0.4 # Generation temperature (0.0-1.0) - proxy: "" # Example: "socks5://127.0.0.1:1081" or "http://127.0.0.1:8080" or leave empty for no proxy - -# Tavily Search Configuration -tavily: - api_key: "your-tavily-api-key-here" # Required: Your Tavily API key - api_base_url: "https://api.tavily.com" # Tavily API base URL - -# Search Settings +# SGR Deep Research Agent - Configuration Template +# Copy this file to config.yaml and fill in your data + +# LLM Configuration +llm: + api_key: "your-openai-api-key-here" # Your OpenAI API key + base_url: "https://api.openai.com/v1" # API base URL + model: "gpt-4o-mini" # Model name + max_tokens: 8000 # Max output tokens + temperature: 0.4 # Temperature (0.0-1.0) + # proxy: "socks5://127.0.0.1:1081" # Optional proxy (socks5:// or http://) + +# Search Configuration (Tavily) search: - max_results: 10 # Maximum number of search results - -# Scraping Settings -scraping: - enabled: false # Enable full text scraping of found pages - max_pages: 5 # Maximum pages to scrape per search - content_limit: 1500 # Character limit for full content per source + tavily_api_key: "your-tavily-api-key-here" # Tavily API key (get at tavily.com) + tavily_api_base_url: "https://api.tavily.com" # Tavily API URL + max_results: 10 # Max search results + max_pages: 5 # Max pages to scrape + content_limit: 1500 # Content char limit per source # Execution Settings execution: - max_steps: 6 # Maximum number of execution steps - reports_dir: "reports" # Directory for saving reports - logs_dir: "logs" # Directory for saving reports - -# Prompts Settings + max_steps: 6 # Max execution steps + max_clarifications: 3 # Max clarification requests + max_iterations: 10 # Max iterations per step + max_searches: 4 # Max search operations + mcp_context_limit: 15000 # Max context length from MCP server response + logs_dir: "logs" # Directory for saving agent execution logs + reports_dir: "reports" # Directory for saving agent reports + +# Prompts Configuration prompts: - prompts_dir: "prompts" # Directory with prompts - system_prompt_file: "system_prompt.txt" # System prompt file - -# Logging Settings -logging: - config_file: "logging_config.yaml" # Logging configuration file path + # Option 1: Use file paths (absolute or relative to project root) + system_prompt_file: "sgr_deep_research/core/prompts/system_prompt.txt" + initial_user_request_file: "sgr_deep_research/core/prompts/initial_user_request.txt" + clarification_response_file: "sgr_deep_research/core/prompts/clarification_response.txt" -mcp: - - # Limit on the result of MCP tool invocation. - # A balanced constant value: not too large to avoid filling the entire context window with potentially unimportant data, - # yet not too small to ensure critical information from a single MCP fits through - context_limit: 15000 + # Option 2: Provide prompts directly as strings (uncomment to use) + # system_prompt_str: "Your custom system prompt here..." + # initial_user_request_str: "Your custom initial request template..." + # clarification_response_str: "Your custom clarification template..." - # https://gofastmcp.com/clients/transports#mcp-json-configuration-transport - transport_config: - mcpServers: - deepwiki: - url: "https://mcp.deepwiki.com/mcp" + # Note: If both file and string are provided, string takes precedence - context7: - url: "https://mcp.context7.com/mcp" +# MCP (Model Context Protocol) Configuration +mcp: + mcpServers: + deepwiki: + url: "https://mcp.deepwiki.com/mcp" + + # Add more MCP servers here: + # your_server: + # url: "https://your-mcp-server.com/mcp" + # headers: + # Authorization: "Bearer your-token" diff --git a/docs/QWEN3_THINKING_SUPPORT.md b/docs/QWEN3_THINKING_SUPPORT.md new file mode 100644 index 0000000..130f179 --- /dev/null +++ b/docs/QWEN3_THINKING_SUPPORT.md @@ -0,0 +1,256 @@ +# Поддержка Qwen3-Thinking моделей в SGR Deep Research + +## Обзор + +Thinking модели семейства Qwen3 не поддерживают напрямую **Structured Output (SO)**, но отлично работают через **Function Calling (FC)**. Эти модели выдают промежуточные рассуждения вместе с результатами вызова инструментов, что требует специальной обработки для извлечения структурированных данных. + +Данная реализация добавляет полную поддержку Qwen3-thinking моделей в рамках архитектуры SGR, обеспечивая универсальность работы как с instruct, так и с thinking моделями. + +## Ключевые компоненты + +### 1. Утилита конвертации Pydantic в Tools (pydantic_convert.py) + +Модуль для автоматического преобразования Pydantic-моделей в формат OpenAI Function Calling: + +```python +from sgr_deep_research.core.utils.pydantic_convert import pydantic_to_tools + +# Конвертация Pydantic-модели в tools формат +tools = pydantic_to_tools( + model=MyPydanticModel, + name="my_function", + description="Описание функции" +) +``` + +**Возможности:** +- Автоматическое формирование JSON Schema из Pydantic-моделей +- Поддержка вложенных моделей +- Обработка `Literal`, `Optional`, `Union`, `List`, `Dict` +- Поддержка constraints (min/max, length, pattern) + +### 2. Адаптер для Qwen3-Thinking (qwen3_thinking_adapter.py) + +Извлекает структурированные ответы из "грязного" вывода thinking-моделей с использованием трех стратегий: + +1. **tool_calls с JSON в arguments** - чистый JSON или JSON с рассуждениями +2. **content с тегами `...`** - приоритетный формат +3. **content с чистым JSON** - fallback вариант + +```python +from sgr_deep_research.core.adapters.qwen3_thinking_adapter import extract_structured_response + +# Извлечение структурированного ответа +result = extract_structured_response( + response_message=api_response.choices[0].message.model_dump(), + schema_class=MyPydanticModel, + tool_name="my_function" +) +``` + +**Особенности:** +- Robustный парсинг с множественными стратегиями +- Поддержка различных форматов вывода thinking-моделей +- Детальная диагностика ошибок валидации + +### 3. SGRQwen3ThinkingAgent + +Полноценный SGR агент, адаптированный для работы с thinking-моделями: + +```python +from sgr_deep_research.core.agents.sgr_qwen3_thinking_agent import SGRQwen3ThinkingAgent + +agent = SGRQwen3ThinkingAgent( + task="Ваша исследовательская задача", + openai_client=openai_client, + llm_config=config.llm, + prompts_config=config.prompts, + execution_config=config.execution, + toolkit=[WebSearchTool, CreateReportTool, FinalAnswerTool] +) + +await agent.execute() +``` + +**Отличия от стандартного SGRResearchAgent:** +- Использует Function Calling вместо Structured Output +- Модифицированный системный промпт с инструкциями для thinking-моделей +- Интеграция с адаптером для извлечения структурированных ответов +- Сохранение всей архитектуры SGR (reasoning → action → evaluation) + +## Конфигурация + +### Базовая конфигурация (config.yaml) + +```yaml +llm: + api_key: "your-api-key" + base_url: "https://api.your-provider.com/v1" # vLLM endpoint + model: "Qwen/Qwen3-14B-thinking" # или другая thinking модель + max_tokens: 8000 + temperature: 0.4 + proxy: "socks5://127.0.0.1:1081" # опционально + +search: + tavily_api_key: "your-tavily-key" + max_results: 10 + +execution: + max_iterations: 10 + max_searches: 4 + max_clarifications: 3 +``` + +### Определение агента (agents.yaml) + +```yaml +agents: + - name: "qwen3_thinking_agent" + base_class: "SGRQwen3ThinkingAgent" + + llm: + model: "Qwen/Qwen3-14B-thinking" + temperature: 0.3 + max_tokens: 12000 + + execution: + max_iterations: 15 + max_searches: 6 + + tools: + - "WebSearchTool" + - "ExtractPageContentTool" + - "CreateReportTool" + - "ClarificationTool" + - "FinalAnswerTool" +``` + +## Использование + +### Базовый пример + +```python +import asyncio +import httpx +from openai import AsyncOpenAI +from sgr_deep_research.core.agent_config import GlobalConfig +from sgr_deep_research.core.agents.sgr_qwen3_thinking_agent import SGRQwen3ThinkingAgent +from sgr_deep_research.core.tools import ( + WebSearchTool, + CreateReportTool, + FinalAnswerTool +) + +async def main(): + # Загрузка конфигурации + config = GlobalConfig.from_yaml("config.yaml") + + # Создание OpenAI клиента + client_kwargs = { + "base_url": config.llm.base_url, + "api_key": config.llm.api_key + } + if config.llm.proxy: + client_kwargs["http_client"] = httpx.AsyncClient(proxy=config.llm.proxy) + + openai_client = AsyncOpenAI(**client_kwargs) + + # Создание агента + agent = SGRQwen3ThinkingAgent( + task="Найди последние новости о Schema-Guided Reasoning", + openai_client=openai_client, + llm_config=config.llm, + prompts_config=config.prompts, + execution_config=config.execution, + toolkit=[WebSearchTool, CreateReportTool, FinalAnswerTool], + ) + + # Выполнение + await agent.execute() + +if __name__ == "__main__": + asyncio.run(main()) +``` + +### Использование через API + +```bash +# Запуск API сервера +uv run python sgr_deep_research + +# Использование агента +curl -X POST "http://0.0.0.0:8010/v1/chat/completions" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "qwen3_thinking_agent", + "messages": [ + { + "role": "user", + "content": "Исследуй концепцию Schema-Guided Reasoning" + } + ] + }' +``` + +## Технические детали + +### Формат ответа thinking-моделей + +Thinking модели могут возвращать данные в нескольких форматах: + +**Формат 1: tool_calls с чистым JSON** +```json +{ + "tool_calls": [{ + "type": "function", + "function": { + "name": "next_step", + "arguments": "{\"reasoning_steps\": [...], \"function\": {...}}" + } + }] +} +``` + +**Формат 2: content с тегами tool_call** +``` + +Промежуточные рассуждения модели... + + + +{"reasoning_steps": ["step 1", "step 2"], "function": {"tool_name_discriminator": "web_search_tool", ...}} + +``` + +**Формат 3: content с чистым JSON** +```json +{ + "reasoning_steps": ["step 1", "step 2"], + "current_situation": "...", + "function": { + "tool_name_discriminator": "web_search_tool", + "query": "..." + } +} +``` + +### Системный промпт для thinking-моделей + +Агент использует специально адаптированный системный промпт: + +``` +{base_system_prompt} + +## Response Schema for Thinking Models: +You must work strictly according to the following Pydantic schema: + +```python +{dynamic_schema} +``` + +## CRITICAL RULES for Thinking Models: +- You may reason and think through the problem first in your internal monologue +- After reasoning, you MUST provide valid JSON matching the schema above +- Wrap your final JSON in ... tags for clarity +- All text fields in JSON must be concise and focused +``` diff --git a/docs/WIKI.md b/docs/WIKI.md index 7ca4dc0..aa8e8f7 100644 --- a/docs/WIKI.md +++ b/docs/WIKI.md @@ -511,9 +511,9 @@ execution: # Prompts Settings prompts: - prompts_dir: "prompts" # Directory with prompts - tool_function_prompt_file: "tool_function_prompt.txt" # Tool function prompt file - system_prompt_file: "system_prompt.txt" # System prompt file + system_prompt_file: "prompts/system_prompt.txt" # Path to system prompt file + initial_user_request_file: "prompts/initial_user_request.txt" # Path to initial user request file + clarification_response_file: "prompts/clarification_response.txt" # Path to clarification response file ``` ### Server Configuration diff --git a/examples/qwen3_thinking_example.py b/examples/qwen3_thinking_example.py new file mode 100644 index 0000000..875d15c --- /dev/null +++ b/examples/qwen3_thinking_example.py @@ -0,0 +1,117 @@ +"""Пример использования SGRQwen3ThinkingAgent для работы с thinking моделями qwen3. + +Показывает, как использовать агента с моделями qwen3-thinking, которые работают через Function Calling вместо Structured Output. +""" + +import asyncio +import logging + +import httpx +from openai import AsyncOpenAI + +from sgr_deep_research.core.agent_config import GlobalConfig +from sgr_deep_research.core.agents.sgr_qwen3_thinking_agent import SGRQwen3ThinkingAgent +from sgr_deep_research.core.tools import ( + ClarificationTool, + CreateReportTool, + FinalAnswerTool, + WebSearchTool, +) + +logging.basicConfig(level=logging.INFO) + + +async def main(): + """Основной пример использования SGRQwen3ThinkingAgent.""" + + # 1. Загружаем глобальную конфигурацию из config.yaml + config = GlobalConfig.from_yaml("config.yaml") + + # 2. Создаем AsyncOpenAI клиент + client_kwargs = { + "base_url": config.llm.base_url, + "api_key": config.llm.api_key + } + if config.llm.proxy: + client_kwargs["http_client"] = httpx.AsyncClient(proxy=config.llm.proxy) + + openai_client = AsyncOpenAI(**client_kwargs) + + # 3. Определяем задачу для агента + task = ( + "Найди информацию о Schema-Guided Reasoning (SGR) и объясни, " + "как эта концепция применяется в современных LLM агентах" + ) + + # 4. Создаем агента с нужными инструментами + agent = SGRQwen3ThinkingAgent( + task=task, + openai_client=openai_client, + llm_config=config.llm, + prompts_config=config.prompts, + execution_config=config.execution, + toolkit=[ + WebSearchTool, + CreateReportTool, + ClarificationTool, + FinalAnswerTool, + ], + ) + + # 5. Запускаем выполнение задачи + print(f"Запуск агента с задачей: {task}\n") + await agent.execute() + + +async def example_with_clarification(): + """Пример с использованием уточнений.""" + + config = GlobalConfig.from_yaml("config.yaml") + + client_kwargs = { + "base_url": config.llm.base_url, + "api_key": config.llm.api_key + } + if config.llm.proxy: + client_kwargs["http_client"] = httpx.AsyncClient(proxy=config.llm.proxy) + + openai_client = AsyncOpenAI(**client_kwargs) + + task = "Исследуй технологию, о которой я думаю" + + agent = SGRQwen3ThinkingAgent( + task=task, + openai_client=openai_client, + llm_config=config.llm, + prompts_config=config.prompts, + execution_config=config.execution, + toolkit=[ + WebSearchTool, + ClarificationTool, + FinalAnswerTool, + ], + ) + + # Запускаем агента в отдельной задаче + agent_task = asyncio.create_task(agent.execute()) + + # Ждем, пока агент запросит уточнение + await asyncio.sleep(5) + + # Предоставляем уточнение + if agent._context.state.value == "waiting_for_clarification": + clarification = "Я думаю о Schema-Guided Reasoning (SGR)" + await agent.provide_clarification(clarification) + + # Ждем завершения выполнения + await agent_task + + print("\nЗадача с уточнением выполнена") + + +if __name__ == "__main__": + # Запускаем основной пример + asyncio.run(main()) + + # Раскомментируйте для запуска примера с уточнением + # asyncio.run(example_with_clarification()) diff --git a/logging_config.yaml b/logging_config.yaml index 88d663d..657086f 100644 --- a/logging_config.yaml +++ b/logging_config.yaml @@ -10,11 +10,6 @@ formatters: format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s' handlers: - console: - class: logging.StreamHandler - level: INFO - formatter: standard - stream: ext://sys.stdout console_error: class: logging.StreamHandler @@ -22,6 +17,13 @@ handlers: formatter: standard stream: ext://sys.stderr + + console: + class: logging.StreamHandler + level: INFO + formatter: standard + stream: ext://sys.stdout + file: class: logging.handlers.RotatingFileHandler level: DEBUG diff --git a/pyproject.toml b/pyproject.toml index 33e0cc0..57e9e7c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,6 @@ dependencies = [ "youtube-transcript-api>=0.6.0", # Configuration and utilities - конфигурация и утилиты "PyYAML>=6.0", - "envyaml>=1.10.0", "python-dateutil>=2.8.0", "pydantic-settings>=2.10.1", "fastapi>=0.116.1", diff --git a/services/api_service/requirements.txt b/services/api_service/requirements.txt index dc8235c..f3c6951 100644 --- a/services/api_service/requirements.txt +++ b/services/api_service/requirements.txt @@ -59,8 +59,6 @@ email-validator==2.3.0 # via # jambo # pydantic -envyaml==1.10.211231 - # via sgr-deep-research (pyproject.toml) exceptiongroup==1.3.0 # via fastmcp fastapi==0.119.0 @@ -188,7 +186,6 @@ pytz==2025.2 pyyaml==6.0.3 # via # sgr-deep-research (pyproject.toml) - # envyaml # jsonschema-path referencing==0.36.2 # via @@ -247,6 +244,8 @@ trafilatura==2.0.0 # via sgr-deep-research (pyproject.toml) typing-extensions==4.15.0 # via + # anyio + # exceptiongroup # fastapi # openai # openapi-core diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 7062d95..ee1947f 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -11,6 +11,7 @@ services: volumes: - ../sgr_deep_research:/app/sgr_deep_research:ro - ../config.yaml:/app/config.yaml:ro + - ../agents.yaml:/app/agents.yaml:ro - ../logging_config.yaml:/app/logging_config.yaml:ro - ./logs:/app/logs - ./reports:/app/reports diff --git a/sgr_deep_research/__init__.py b/sgr_deep_research/__init__.py index bf0472e..3a7d24b 100644 --- a/sgr_deep_research/__init__.py +++ b/sgr_deep_research/__init__.py @@ -6,7 +6,6 @@ from sgr_deep_research.api import * # noqa: F403 from sgr_deep_research.core import * # noqa: F403 -from sgr_deep_research.services import * # noqa: F403 __version__ = "0.2.5" __author__ = "sgr-deep-research-team" diff --git a/sgr_deep_research/__main__.py b/sgr_deep_research/__main__.py index ae9c587..9571ec1 100644 --- a/sgr_deep_research/__main__.py +++ b/sgr_deep_research/__main__.py @@ -1,17 +1,16 @@ """Основная точка входа для SGR Deep Research API сервера.""" -import argparse import logging -import os from contextlib import asynccontextmanager import uvicorn from fastapi import FastAPI -from sgr_deep_research import __version__ +from sgr_deep_research import AgentFactory, __version__ from sgr_deep_research.api.endpoints import router -from sgr_deep_research.services import MCP2ToolConverter -from sgr_deep_research.settings import setup_logging +from sgr_deep_research.core import AgentRegistry, ToolRegistry +from sgr_deep_research.core.agent_config import GlobalConfig +from sgr_deep_research.settings import ServerConfig, setup_logging setup_logging() logger = logging.getLogger(__name__) @@ -19,26 +18,22 @@ @asynccontextmanager async def lifespan(app: FastAPI): - await MCP2ToolConverter().build_tools_from_mcp() + for tool in ToolRegistry.list_items(): + logger.info(f"Tool registered: {tool.__name__}") + for agent in AgentRegistry.list_items(): + logger.info(f"Agent registered: {agent.__name__}") + for defn in AgentFactory.get_definitions_list(): + logger.info(f"Agent definition loaded: {defn}") yield -app = FastAPI(title="SGR Deep Research API", version=__version__, lifespan=lifespan) -app.include_router(router) - - def main(): """Запуск FastAPI сервера.""" - - parser = argparse.ArgumentParser(description="SGR Deep Research Server") - parser.add_argument( - "--host", type=str, dest="host", default=os.environ.get("HOST", "0.0.0.0"), help="Хост для прослушивания" - ) - parser.add_argument( - "--port", type=int, dest="port", default=int(os.environ.get("PORT", 8010)), help="Порт для прослушивания" - ) - args = parser.parse_args() - + args = ServerConfig() + GlobalConfig.from_yaml(args.config_file) + GlobalConfig.definitions_from_yaml(args.agents_file) + app = FastAPI(title="SGR Deep Research API", version=__version__, lifespan=lifespan) + app.include_router(router) uvicorn.run(app, host=args.host, port=args.port, log_level="info") diff --git a/sgr_deep_research/api/endpoints.py b/sgr_deep_research/api/endpoints.py index 4b37be1..a749c11 100644 --- a/sgr_deep_research/api/endpoints.py +++ b/sgr_deep_research/api/endpoints.py @@ -5,16 +5,15 @@ from fastapi.responses import StreamingResponse from sgr_deep_research.api.models import ( - AGENT_MODEL_MAPPING, AgentListItem, AgentListResponse, - AgentModel, AgentStateResponse, ChatCompletionRequest, ClarificationRequest, HealthResponse, ) -from sgr_deep_research.core.agents import BaseAgent +from sgr_deep_research.core import BaseAgent +from sgr_deep_research.core.agent_factory import AgentFactory from sgr_deep_research.core.models import AgentStatesEnum logger = logging.getLogger(__name__) @@ -63,13 +62,17 @@ async def get_agents_list(): @router.get("/v1/models") async def get_available_models(): """Get list of available agent models.""" - return { - "data": [ - {"id": model.value, "object": "model", "created": 1234567890, "owned_by": "sgr-deep-research"} - for model in AgentModel - ], - "object": "list", - } + models_data = [ + { + "id": agent_def.name, + "object": "model", + "created": 1234567890, + "owned_by": "sgr-deep-research", + } + for agent_def in AgentFactory.get_definitions_list() + ] + + return {"data": models_data, "object": "list"} def extract_user_content_from_messages(messages): @@ -131,19 +134,17 @@ async def create_chat_completion(request: ChatCompletionRequest): try: task = extract_user_content_from_messages(request.messages) - try: - agent_model = AgentModel(request.model) - except ValueError: + agent_def = next(filter(lambda ad: ad.name == request.model, AgentFactory.get_definitions_list()), None) + if not agent_def: raise HTTPException( status_code=400, - detail=f"Invalid model '{request.model}'. Available models: {[m.value for m in AgentModel]}", + detail=f"Invalid model '{request.model}'. " + f"Available models: {[ad.name for ad in AgentFactory.get_definitions_list()]}", ) + agent = await AgentFactory.create(agent_def, task) + logger.info(f"Created agent '{request.model}' for task: {task[:100]}...") - agent_class = AGENT_MODEL_MAPPING[agent_model] - agent = agent_class(task=task) agents_storage[agent.id] = agent - logger.info(f"Agent {agent.id} ({agent_model.value}) created and stored for task: {task[:100]}...") - _ = asyncio.create_task(agent.execute()) return StreamingResponse( agent.streaming_generator.stream(), @@ -152,9 +153,10 @@ async def create_chat_completion(request: ChatCompletionRequest): "Cache-Control": "no-cache", "Connection": "keep-alive", "X-Agent-ID": str(agent.id), - "X-Agent-Model": agent_model.value, + "X-Agent-Model": request.model, }, ) except ValueError as e: + logger.error(f"Error completion: {e}", exc_info=True) raise HTTPException(status_code=400, detail=str(e)) diff --git a/sgr_deep_research/api/models.py b/sgr_deep_research/api/models.py index a8f824a..9fcf366 100644 --- a/sgr_deep_research/api/models.py +++ b/sgr_deep_research/api/models.py @@ -1,39 +1,10 @@ """OpenAI-compatible models for API endpoints.""" from datetime import datetime -from enum import Enum from typing import Any, Dict, List, Literal from pydantic import BaseModel, Field -from sgr_deep_research.core.agents import ( - SGRAutoToolCallingResearchAgent, - SGRResearchAgent, - SGRSOToolCallingResearchAgent, - SGRToolCallingResearchAgent, - ToolCallingResearchAgent, -) - - -class AgentModel(str, Enum): - """Available agent models for chat completion.""" - - SGR_AGENT = SGRResearchAgent.name - SGR_TOOLS_AGENT = SGRToolCallingResearchAgent.name - SGR_AUTO_TOOLS_AGENT = SGRAutoToolCallingResearchAgent.name - SGR_SO_TOOLS_AGENT = SGRSOToolCallingResearchAgent.name - TOOLS_AGENT = ToolCallingResearchAgent.name - - -# Mapping of agent types to their classes -AGENT_MODEL_MAPPING = { - AgentModel.SGR_AGENT: SGRResearchAgent, - AgentModel.SGR_TOOLS_AGENT: SGRToolCallingResearchAgent, - AgentModel.SGR_AUTO_TOOLS_AGENT: SGRAutoToolCallingResearchAgent, - AgentModel.SGR_SO_TOOLS_AGENT: SGRSOToolCallingResearchAgent, - AgentModel.TOOLS_AGENT: ToolCallingResearchAgent, -} - class ChatMessage(BaseModel): """Chat message.""" @@ -46,9 +17,11 @@ class ChatCompletionRequest(BaseModel): """Request for creating chat completion.""" model: str | None = Field( - default=AgentModel.SGR_AGENT, + default="sgr_tool_calling_agent", description="Agent type or existing agent identifier", - examples=[AgentModel.SGR_AGENT.value], + examples=[ + "sgr_tool_calling_agent", + ], ) messages: List[ChatMessage] = Field(description="List of messages") stream: bool = Field(default=True, description="Enable streaming mode") diff --git a/sgr_deep_research/core/__init__.py b/sgr_deep_research/core/__init__.py index 751ba1a..a3bd063 100644 --- a/sgr_deep_research/core/__init__.py +++ b/sgr_deep_research/core/__init__.py @@ -1,32 +1,46 @@ """Core modules for SGR Deep Research.""" +from sgr_deep_research.core.agent_definition import AgentDefinition +from sgr_deep_research.core.agent_factory import AgentFactory from sgr_deep_research.core.agents import ( # noqa: F403 - BaseAgent, SGRAutoToolCallingResearchAgent, SGRResearchAgent, SGRSOToolCallingResearchAgent, SGRToolCallingResearchAgent, ToolCallingResearchAgent, ) +from sgr_deep_research.core.base_agent import BaseAgent +from sgr_deep_research.core.base_tool import BaseTool, MCPBaseTool from sgr_deep_research.core.models import AgentStatesEnum, ResearchContext, SearchResult, SourceData -from sgr_deep_research.core.prompts import PromptLoader +from sgr_deep_research.core.services import AgentRegistry, MCP2ToolConverter, PromptLoader, ToolRegistry from sgr_deep_research.core.stream import OpenAIStreamingGenerator from sgr_deep_research.core.tools import * # noqa: F403 __all__ = [ # Agents "BaseAgent", + "AgentDefinition", + # Tools + "BaseTool", + "MCPBaseTool", + # Agents "SGRResearchAgent", "SGRToolCallingResearchAgent", "SGRAutoToolCallingResearchAgent", "ToolCallingResearchAgent", "SGRSOToolCallingResearchAgent", + # Factories + "AgentFactory", + # Services + "AgentRegistry", + "ToolRegistry", + "PromptLoader", + "MCP2ToolConverter", # Models "AgentStatesEnum", "ResearchContext", "SearchResult", "SourceData", # Other core modules - "PromptLoader", "OpenAIStreamingGenerator", ] diff --git a/sgr_deep_research/core/adapters/__init__.py b/sgr_deep_research/core/adapters/__init__.py new file mode 100644 index 0000000..ebdfaee --- /dev/null +++ b/sgr_deep_research/core/adapters/__init__.py @@ -0,0 +1,13 @@ +"""Адаптеры для работы с различными моделями.""" + +from sgr_deep_research.core.adapters.qwen3_thinking_adapter import ( + extract_structured_response, + robust_json_extract, + extract_tool_call_from_tags, +) + +__all__ = [ + "extract_structured_response", + "robust_json_extract", + "extract_tool_call_from_tags", +] diff --git a/sgr_deep_research/core/adapters/qwen3_thinking_adapter.py b/sgr_deep_research/core/adapters/qwen3_thinking_adapter.py new file mode 100644 index 0000000..33971a6 --- /dev/null +++ b/sgr_deep_research/core/adapters/qwen3_thinking_adapter.py @@ -0,0 +1,200 @@ +"""Адаптер для работы с qwen3-thinking моделями через Function Calling. + +Thinking модели семейства qwen3 не поддерживают напрямую Structured Output (SO), +но могут работать через Function Calling (FC). Этот адаптер извлекает структурированные +данные из ответов thinking-моделей, которые содержат промежуточные рассуждения вместе +с результатами вызова инструментов. +""" + +import json +import logging +import re +from typing import Optional, Type + +from pydantic import BaseModel + +logger = logging.getLogger(__name__) + + +def robust_json_extract(text: str) -> list[dict]: + """Извлекает все возможные JSON-объекты из 'грязного' текста. + + Args: + text: Текст, содержащий JSON-объекты + + Returns: + Список извлеченных JSON-объектов + """ + if not text or not text.strip(): + return [] + + results = [] + seen = set() + + # 1. Markdown-блоки ```json ... ``` + json_blocks = re.findall(r"```(?:json)?\s*({.*?})\s*```", text, re.DOTALL | re.IGNORECASE) + candidates = json_blocks[:] + + # 2. Все {...} конструкции (нежадно и жадно) + candidates += re.findall(r"\{.*?\}", text, re.DOTALL) + candidates += re.findall(r"\{.*\}", text, re.DOTALL) + + # Убираем дубли, сортируем по длине (длиннее — раньше) + unique = [] + for c in candidates: + c_clean = c.strip() + if c_clean and c_clean not in seen: + seen.add(c_clean) + unique.append(c_clean) + unique.sort(key=len, reverse=True) + + for cand in unique: + try: + obj = json.loads(cand) + if isinstance(obj, dict): + results.append(obj) + except (json.JSONDecodeError, ValueError): + continue + return results + + +def extract_tool_call_from_tags(text: str) -> Optional[dict]: + """Извлекает JSON из тегов .... + + Args: + text: Текст, содержащий теги tool_call + + Returns: + Извлеченный JSON-объект или None + """ + pattern = r'\s*(\{.*?\})\s*' + matches = re.findall(pattern, text, re.DOTALL) + if not matches: + return None + + # Берём последний (самый актуальный) tool_call + for match in reversed(matches): + try: + return json.loads(match) + except json.JSONDecodeError: + continue + return None + + +def extract_structured_response( + response_message: dict, + schema_class: Type[BaseModel], + tool_name: str +) -> BaseModel: + """Извлекает структурированный ответ из ответа thinking-модели. + + Thinking модели могут возвращать данные в разных форматах: + 1. В поле tool_calls[].function.arguments (чистый JSON или с рассуждениями) + 2. В поле content с тегами ... + 3. В поле content как чистый JSON + + Args: + response_message: Сообщение от API (choices[0].message) + schema_class: Pydantic-класс для валидации ответа + tool_name: Имя инструмента + + Returns: + Валидированный объект Pydantic-модели + + Raises: + RuntimeError: Если не удалось извлечь или валидировать ответ + """ + content = (response_message.get('content') or '').strip() + tool_calls = response_message.get('tool_calls') or [] + + validation_errors = [] + + # === Стратегия 1: tool_calls с чистым JSON в arguments === + for tc in tool_calls: + if tc.get('type') != 'function': + continue + + func = tc.get('function', {}) + name = func.get('name') + if name != tool_name: + continue + + raw_args = func.get('arguments', '') + if not raw_args: + continue + + # Парсим arguments + if isinstance(raw_args, str): + # Сначала пробуем как чистый JSON + try: + args = json.loads(raw_args) + validated = schema_class.model_validate(args) + logger.debug(f"Успешно извлечен ответ из tool_calls.arguments (JSON)") + return validated + except json.JSONDecodeError: + # Может быть рассуждения + JSON, извлекаем все JSON-объекты + json_objects = robust_json_extract(raw_args) + for obj in json_objects: + try: + validated = schema_class.model_validate(obj) + logger.debug(f"Успешно извлечен ответ из tool_calls.arguments (extracted JSON)") + return validated + except (ValueError, TypeError) as e: + validation_errors.append(f"tool_calls.arguments JSON: {type(e).__name__}: {e}") + except (ValueError, TypeError) as e: + validation_errors.append(f"tool_calls.arguments: {type(e).__name__}: {e}") + elif isinstance(raw_args, dict): + try: + validated = schema_class.model_validate(raw_args) + logger.debug(f"Успешно извлечен ответ из tool_calls.arguments (dict)") + return validated + except (ValueError, TypeError) as e: + validation_errors.append(f"tool_calls.arguments dict: {type(e).__name__}: {e}") + + # === Стратегия 2: content с тегами ... (ПРИОРИТЕТ!) === + if content: + tool_call_obj = extract_tool_call_from_tags(content) + if tool_call_obj: + # Случай 1: {"name": "...", "arguments": {...}} + if tool_call_obj.get('name') == tool_name and 'arguments' in tool_call_obj: + args = tool_call_obj['arguments'] + try: + validated = schema_class.model_validate(args) + logger.debug(f"Успешно извлечен ответ из с name") + return validated + except (ValueError, TypeError) as e: + validation_errors.append(f" with name: {type(e).__name__}: {e}") + + # Случай 2: прямой JSON без обёртки {"name": ...} + try: + validated = schema_class.model_validate(tool_call_obj) + logger.debug(f"Успешно извлечен ответ из прямой JSON") + return validated + except (ValueError, TypeError) as e: + validation_errors.append(f" direct: {type(e).__name__}: {e}") + + # === Стратегия 3: content с чистым JSON === + if content: + json_objects = robust_json_extract(content) + for obj in json_objects: + try: + validated = schema_class.model_validate(obj) + logger.debug(f"Успешно извлечен ответ из content JSON") + return validated + except (ValueError, TypeError) as e: + validation_errors.append(f"content JSON object: {type(e).__name__}: {e}") + + # === Ничего не сработало === + error_details = [ + f"Не удалось извлечь структурированный ответ для {tool_name}", + f"Content length: {len(content)}", + f"Content preview: {content[:300]!r}", + f"Tool calls count: {len(tool_calls)}", + f"Tool calls: {tool_calls!r}", + f"Validation attempts: {len(validation_errors)}", + ] + if validation_errors: + error_details.append("Validation errors:") + error_details.extend(f" - {err}" for err in validation_errors[:5]) + + raise RuntimeError("\n".join(error_details)) diff --git a/sgr_deep_research/core/agent_config.py b/sgr_deep_research/core/agent_config.py new file mode 100644 index 0000000..0de32b5 --- /dev/null +++ b/sgr_deep_research/core/agent_config.py @@ -0,0 +1,51 @@ +from pathlib import Path +from typing import ClassVar, Self + +import yaml +from pydantic_settings import BaseSettings, SettingsConfigDict + +from sgr_deep_research.core.agent_definition import AgentConfig, Definitions + + +class GlobalConfig(BaseSettings, AgentConfig, Definitions): + _instance: ClassVar[Self | None] = None + _initialized: ClassVar[bool] = False + + def __new__(cls, *args, **kwargs): + if cls._instance is None: + cls._instance = super().__new__(cls) + return cls._instance + + def __init__(self, *args, **kwargs): + if self._initialized: + return + super().__init__(*args, **kwargs) + self.__class__._initialized = True + + model_config = SettingsConfigDict( + env_prefix="SGR__", + extra="ignore", + case_sensitive=False, + env_nested_delimiter="__", + ) + + @classmethod + def from_yaml(cls, yaml_path: str) -> Self: + yaml_path = Path(yaml_path) + if not yaml_path.exists(): + raise FileNotFoundError(f"Configuration file not found: {yaml_path}") + config_data = yaml.safe_load(yaml_path.read_text(encoding="utf-8")) + if cls._instance is None: + cls._instance = cls(**config_data) + else: + cls._initialized = False + cls._instance = cls(**config_data, agents=cls._instance.agents) + return cls._instance + + @classmethod + def definitions_from_yaml(cls, agents_yaml_path: str) -> Self: + agents_yaml_path = Path(agents_yaml_path) + if not agents_yaml_path.exists(): + raise FileNotFoundError(f"Agents definitions file not found: {agents_yaml_path}") + cls._instance.agents = Definitions(**yaml.safe_load(agents_yaml_path.read_text(encoding="utf-8"))).agents + return cls._instance diff --git a/sgr_deep_research/core/agent_definition.py b/sgr_deep_research/core/agent_definition.py new file mode 100644 index 0000000..9923124 --- /dev/null +++ b/sgr_deep_research/core/agent_definition.py @@ -0,0 +1,177 @@ +import logging +import os +from functools import cached_property +from pathlib import Path +from typing import Any, Self + +import yaml +from fastmcp.mcp_config import MCPConfig +from pydantic import BaseModel, Field, FilePath, computed_field, model_validator + +logger = logging.getLogger(__name__) + + +class LLMConfig(BaseModel): + api_key: str | None = Field(default=None, description="API key") + base_url: str = Field(default="https://api.openai.com/v1", description="Base URL") + model: str = Field(default="gpt-4o-mini", description="Model to use") + max_tokens: int = Field(default=8000, description="Maximum number of output tokens") + temperature: float = Field(default=0.4, ge=0.0, le=1.0, description="Generation temperature") + proxy: str | None = Field( + default=None, description="Proxy URL (e.g., socks5://127.0.0.1:1081 or http://127.0.0.1:8080)" + ) + + +class SearchConfig(BaseModel): + tavily_api_key: str | None = Field(default=None, description="Tavily API key") + tavily_api_base_url: str = Field(default="https://api.tavily.com", description="Tavily API base URL") + + max_results: int = Field(default=10, ge=1, description="Maximum number of search results") + max_pages: int = Field(default=5, gt=0, description="Maximum pages to scrape") + content_limit: int = Field(default=1500, gt=0, description="Content character limit per source") + + +class PromptsConfig(BaseModel): + system_prompt_file: FilePath | None = Field( + default=os.path.join(os.path.dirname(__file__), "prompts/system_prompt.txt"), + description="Path to system prompt file", + ) + initial_user_request_file: FilePath | None = Field( + default=os.path.join(os.path.dirname(__file__), "prompts/initial_user_request.txt"), + description="Path to initial user request file", + ) + clarification_response_file: FilePath | None = Field( + default=os.path.join(os.path.dirname(__file__), "prompts/clarification_response.txt"), + description="Path to clarification response file", + ) + system_prompt_str: str | None = None + initial_user_request_str: str | None = None + clarification_response_str: str | None = None + + @computed_field + @cached_property + def system_prompt(self) -> str: + return self.system_prompt_str or self._load_prompt_file(self.system_prompt_file) + + @computed_field + @cached_property + def initial_user_request(self) -> str: + return self.initial_user_request_str or self._load_prompt_file(self.initial_user_request_file) + + @computed_field + @cached_property + def clarification_response(self) -> str: + return self.clarification_response_str or self._load_prompt_file(self.clarification_response_file) + + @staticmethod + def _load_prompt_file(file_path: str | None) -> str | None: + """Load prompt content from a file.""" + return Path(file_path).read_text(encoding="utf-8") + + @model_validator(mode="after") + def defaults_validator(self): + for attr, file_attr in zip( + ["system_prompt_str", "initial_user_request_str", "clarification_response_str"], + ["system_prompt_file", "initial_user_request_file", "clarification_response_file"], + ): + field = getattr(self, attr) + file_field: FilePath = getattr(self, file_attr) + if not field and not file_field: + raise ValueError(f"{attr} or {file_attr} must be provided") + if file_field: + project_path = Path(file_field) + if not project_path.exists(): + raise FileNotFoundError(f"Prompt file '{project_path.absolute()}' not found") + return self + + def __repr__(self) -> str: + return ( + f"PromptsConfig(system_prompt='{self.system_prompt[:100]}...', " + f"initial_user_request='{self.initial_user_request[:100]}...', " + f"clarification_response='{self.clarification_response[:100]}...')" + ) + + +class ExecutionConfig(BaseModel, extra="allow"): + """Execution parameters and limits for agents. + + You can add any additional fields as needed. + """ + + max_steps: int = Field(default=6, gt=0, description="Maximum number of execution steps") + max_clarifications: int = Field(default=3, ge=0, description="Maximum number of clarifications") + max_iterations: int = Field(default=10, gt=0, description="Maximum number of iterations") + max_searches: int = Field(default=4, ge=0, description="Maximum number of searches") + mcp_context_limit: int = Field(default=15000, gt=0, description="Maximum context length from MCP server response") + + logs_dir: str = Field(default="logs", description="Directory for saving bot logs") + reports_dir: str = Field(default="reports", description="Directory for saving reports") + + +class AgentConfig(BaseModel): + llm: LLMConfig = Field(default_factory=LLMConfig, description="LLM settings") + search: SearchConfig | None = Field(default=None, description="Search settings") + execution: ExecutionConfig = Field(default_factory=ExecutionConfig, description="Execution settings") + prompts: PromptsConfig = Field(default_factory=PromptsConfig, description="Prompts settings") + mcp: MCPConfig = Field(default_factory=MCPConfig, description="MCP settings") + + +class AgentDefinition(AgentConfig): + """Definition of a custom agent. + + Agents can override global settings by providing: + - llm: dict with keys matching LLMConfig (api_key, base_url, model, etc.) + - prompts: dict with keys matching PromptsConfig (system_prompt_file, etc.) + - ExecutionConfig: execution parameters and limits + - tools: list of tool names to include + """ + + name: str = Field(description="Unique agent name/ID") + # ToDo: not sure how to type this properly and avoid circular imports + base_class: type[Any] | str = Field(description="Agent class name") + tools: list[type[Any] | str] = Field(default_factory=list, description="List of tool names to include") + + @model_validator(mode="before") + def default_config_override_validator(cls, data): + print(data) + from sgr_deep_research.core.agent_config import GlobalConfig + + data["llm"] = GlobalConfig().llm.model_copy(update=data.get("llm", {})).model_dump() + data["search"] = ( + GlobalConfig().search.model_copy(update=data.get("search", {})).model_dump() + if GlobalConfig().search + else None + ) + data["execution"] = GlobalConfig().execution.model_copy(update=data.get("execution", {})).model_dump() + return data + + @model_validator(mode="after") + def necessary_fields_validator(self) -> Self: + if self.llm.api_key is None: + raise ValueError(f"LLM API key is not provided for agent '{self.name}'") + if self.search and self.search.tavily_api_key is None: + raise ValueError(f"Search API key is not provided for agent '{self.name}'") + if not self.tools: + raise ValueError(f"Tools are not provided for agent '{self.name}'") + return self + + def __str__(self) -> str: + base_class_name = self.base_class.__name__ if isinstance(self.base_class, type) else self.base_class + tool_names = [t.__name__ if isinstance(t, type) else t for t in self.tools] + return ( + f"AgentDefinition(name='{self.name}', " + f"base_class={base_class_name}, " + f"tools={tool_names}, " + f"execution={self.execution}), " + ) + + @classmethod + def from_yaml(cls, yaml_path: str) -> Self: + try: + return cls(**yaml.safe_load(Path(yaml_path).read_text(encoding="utf-8"))) + except FileNotFoundError as e: + raise FileNotFoundError(f"Agent definition file not found: {yaml_path}") from e + + +class Definitions(BaseModel): + agents: list[AgentDefinition] = Field(default_factory=list, description="List of agent definitions") diff --git a/sgr_deep_research/core/agent_factory.py b/sgr_deep_research/core/agent_factory.py new file mode 100644 index 0000000..b7ebfc4 --- /dev/null +++ b/sgr_deep_research/core/agent_factory.py @@ -0,0 +1,122 @@ +"""Agent Factory for dynamic agent creation from definitions.""" + +import logging +from typing import Type, TypeVar + +import httpx +from openai import AsyncOpenAI + +from sgr_deep_research.core.agent_config import GlobalConfig +from sgr_deep_research.core.agent_definition import AgentDefinition, LLMConfig +from sgr_deep_research.core.agents.definitions import get_default_agents_definitions +from sgr_deep_research.core.base_agent import BaseAgent +from sgr_deep_research.core.services import AgentRegistry, MCP2ToolConverter, ToolRegistry + +logger = logging.getLogger(__name__) + +Agent = TypeVar("Agent", bound=BaseAgent) + + +class AgentFactory: + """Factory for creating agent instances from definitions. + + Use AgentRegistry and ToolRegistry to look up agent classes by name + and create instances with the appropriate configuration. + """ + + @classmethod + def _create_client(cls, llm_config: LLMConfig) -> AsyncOpenAI: + """Create OpenAI client from configuration. + + Args: + llm_config: LLM configuration + + Returns: + Configured AsyncOpenAI client + """ + client_kwargs = {"base_url": llm_config.base_url, "api_key": llm_config.api_key} + if llm_config.proxy: + client_kwargs["http_client"] = httpx.AsyncClient(proxy=llm_config.proxy) + + return AsyncOpenAI(**client_kwargs) + + @classmethod + async def create(cls, agent_def: AgentDefinition, task: str) -> Agent: + """Create an agent instance from a definition. + + Args: + agent_def: Agent definition with configuration (classes already resolved) + task: Task for the agent to execute + + Returns: + Created agent instance + + Raises: + ValueError: If agent creation fails + """ + BaseClass: Type[Agent] | None = ( + AgentRegistry.get(agent_def.base_class) if isinstance(agent_def.base_class, str) else agent_def.base_class + ) + if BaseClass is None: + error_msg = ( + f"Agent base class '{agent_def.base_class}' not found in registry.\n" + f"Available base classes: {', '.join([c.__name__ for c in AgentRegistry.list_items()])}\n" + f"To fix this issue:\n" + f" - Check that '{agent_def.base_class}' is spelled correctly in your configuration\n" + f" - Ensure the custom agent classes are imported before creating agents " + f"(otherwise they won't be registered)" + ) + logger.error(error_msg) + raise ValueError(error_msg) + mcp_tools = await MCP2ToolConverter.build_tools_from_mcp(agent_def.mcp) + + tools = [*mcp_tools] + for tool in agent_def.tools: + if isinstance(tool, str): + tool_class = ToolRegistry.get(tool) + if tool_class is None: + error_msg = ( + f"Tool '{tool}' not found in registry.\nAvailable tools: " + f"{', '.join([c.__name__ for c in ToolRegistry.list_items()])}\n" + f" - Ensure the custom tool classes are imported before creating agents " + f"(otherwise they won't be registered)" + ) + logger.error(error_msg) + raise ValueError(error_msg) + else: + tool_class = tool + tools.append(tool_class) + + try: + agent = BaseClass( + task=task, + toolkit=tools, + openai_client=cls._create_client(agent_def.llm), + llm_config=agent_def.llm, + execution_config=agent_def.execution, + prompts_config=agent_def.prompts, + ) + logger.info( + f"Created agent '{agent_def.name}' " + f"using base class '{BaseClass.__name__}' " + f"with {len(agent.toolkit)} tools" + ) + return agent + except Exception as e: + logger.error(f"Failed to create agent '{agent_def.name}': {e}", exc_info=True) + raise ValueError(f"Failed to create agent: {e}") from e + + @classmethod + def get_definitions_list(cls) -> list[AgentDefinition]: + """Get all agent definitions (default + custom from config). + + Returns: + List of agent definitions (default agents + custom agents from config) + """ + config = GlobalConfig() + + all_agents = {agent.name: agent for agent in get_default_agents_definitions()} + if will_be_rewritten := set(all_agents.keys()).intersection({agent.name for agent in config.agents}): + logger.warning(f"Custom agents with names {', '.join(will_be_rewritten)} will override default agents.") + all_agents.update({agent.name: agent for agent in config.agents}) + return list(all_agents.values()) diff --git a/sgr_deep_research/core/agents/__init__.py b/sgr_deep_research/core/agents/__init__.py index 21bf61b..eecc1d9 100644 --- a/sgr_deep_research/core/agents/__init__.py +++ b/sgr_deep_research/core/agents/__init__.py @@ -1,6 +1,6 @@ """Agents module for SGR Deep Research.""" -from sgr_deep_research.core.agents.base_agent import BaseAgent +from sgr_deep_research.core.agents.definitions import get_default_agents_definitions from sgr_deep_research.core.agents.sgr_agent import SGRResearchAgent from sgr_deep_research.core.agents.sgr_auto_tools_agent import SGRAutoToolCallingResearchAgent from sgr_deep_research.core.agents.sgr_so_tools_agent import SGRSOToolCallingResearchAgent @@ -8,10 +8,10 @@ from sgr_deep_research.core.agents.tools_agent import ToolCallingResearchAgent __all__ = [ - "BaseAgent", "SGRResearchAgent", "SGRToolCallingResearchAgent", "SGRAutoToolCallingResearchAgent", "ToolCallingResearchAgent", "SGRSOToolCallingResearchAgent", + "get_default_agents_definitions", ] diff --git a/sgr_deep_research/core/agents/definitions.py b/sgr_deep_research/core/agents/definitions.py new file mode 100644 index 0000000..8302354 --- /dev/null +++ b/sgr_deep_research/core/agents/definitions.py @@ -0,0 +1,55 @@ +import sgr_deep_research.core.tools as tools +from sgr_deep_research.core.agent_definition import AgentDefinition +from sgr_deep_research.core.agents.sgr_agent import SGRResearchAgent +from sgr_deep_research.core.agents.sgr_auto_tools_agent import SGRAutoToolCallingResearchAgent +from sgr_deep_research.core.agents.sgr_so_tools_agent import SGRSOToolCallingResearchAgent +from sgr_deep_research.core.agents.sgr_tools_agent import SGRToolCallingResearchAgent +from sgr_deep_research.core.agents.tools_agent import ToolCallingResearchAgent + +DEFAULT_TOOLKIT = [ + tools.ClarificationTool, + tools.GeneratePlanTool, + tools.AdaptPlanTool, + tools.FinalAnswerTool, + tools.WebSearchTool, + tools.ExtractPageContentTool, + tools.CreateReportTool, +] + + +def get_default_agents_definitions() -> list[AgentDefinition]: + """Get default agent definitions. + + This function creates agent definitions lazily to avoid issues with + configuration initialization order. + + Returns: + List of default agent definitions + """ + return [ + AgentDefinition( + name="sgr_agent", + base_class=SGRResearchAgent, + tools=DEFAULT_TOOLKIT, + ), + AgentDefinition( + name="tool_calling_agent", + base_class=ToolCallingResearchAgent, + tools=DEFAULT_TOOLKIT, + ), + AgentDefinition( + name="sgr_tool_calling_agent", + base_class=SGRToolCallingResearchAgent, + tools=DEFAULT_TOOLKIT, + ), + AgentDefinition( + name="sgr_auto_tool_calling_agent", + base_class=SGRAutoToolCallingResearchAgent, + tools=DEFAULT_TOOLKIT, + ), + AgentDefinition( + name="sgr_so_tool_calling_agent", + base_class=SGRSOToolCallingResearchAgent, + tools=DEFAULT_TOOLKIT, + ), + ] diff --git a/sgr_deep_research/core/agents/sgr_agent.py b/sgr_deep_research/core/agents/sgr_agent.py index d44ffd3..1095522 100644 --- a/sgr_deep_research/core/agents/sgr_agent.py +++ b/sgr_deep_research/core/agents/sgr_agent.py @@ -1,6 +1,9 @@ from typing import Type -from sgr_deep_research.core.agents.base_agent import BaseAgent +from openai import AsyncOpenAI + +from sgr_deep_research.core.agent_definition import ExecutionConfig, LLMConfig, PromptsConfig +from sgr_deep_research.core.base_agent import BaseAgent from sgr_deep_research.core.tools import ( BaseTool, ClarificationTool, @@ -8,15 +11,8 @@ FinalAnswerTool, NextStepToolsBuilder, NextStepToolStub, - ReasoningTool, WebSearchTool, - research_agent_tools, - system_agent_tools, ) -from sgr_deep_research.services import MCP2ToolConverter -from sgr_deep_research.settings import get_config - -config = get_config() class SGRResearchAgent(BaseAgent): @@ -27,26 +23,21 @@ class SGRResearchAgent(BaseAgent): def __init__( self, task: str, + openai_client: AsyncOpenAI, + llm_config: LLMConfig, + prompts_config: PromptsConfig, + execution_config: ExecutionConfig, toolkit: list[Type[BaseTool]] | None = None, - max_clarifications: int = 3, - max_iterations: int = 10, - max_searches: int = 4, ): super().__init__( task=task, + openai_client=openai_client, + llm_config=llm_config, + prompts_config=prompts_config, + execution_config=execution_config, toolkit=toolkit, - max_clarifications=max_clarifications, - max_iterations=max_iterations, ) - - self.toolkit = [ - *system_agent_tools, - *research_agent_tools, - *MCP2ToolConverter().toolkit, - *(toolkit or []), - ] - self.toolkit.remove(ReasoningTool) # we use our own reasoning scheme - self.max_searches = max_searches + self.max_searches = execution_config.max_searches async def _prepare_tools(self) -> Type[NextStepToolStub]: """Prepare tool classes with current context limits.""" @@ -68,11 +59,11 @@ async def _prepare_tools(self) -> Type[NextStepToolStub]: async def _reasoning_phase(self) -> NextStepToolStub: async with self.openai_client.chat.completions.stream( - model=config.openai.model, + model=self.llm_config.model, response_format=await self._prepare_tools(), messages=await self._prepare_context(), - max_tokens=config.openai.max_tokens, - temperature=config.openai.temperature, + max_tokens=self.llm_config.max_tokens, + temperature=self.llm_config.temperature, ) as stream: async for event in stream: if event.type == "chunk": @@ -116,20 +107,3 @@ async def _action_phase(self, tool: BaseTool) -> str: self.streaming_generator.add_chunk_from_str(f"{result}\n") self._log_tool_execution(tool, result) return result - - -if __name__ == "__main__": - import asyncio - - async def main(): - await MCP2ToolConverter().build_tools_from_mcp() - agent = SGRResearchAgent( - task="найди информацию о репозитории на гитхаб sgr-deep-research и ответь на вопрос, " - "какая основная концепция этого репозитория?", - max_iterations=5, - max_clarifications=2, - max_searches=3, - ) - await agent.execute() - - asyncio.run(main()) diff --git a/sgr_deep_research/core/agents/sgr_auto_tools_agent.py b/sgr_deep_research/core/agents/sgr_auto_tools_agent.py index 8749206..5ab073f 100644 --- a/sgr_deep_research/core/agents/sgr_auto_tools_agent.py +++ b/sgr_deep_research/core/agents/sgr_auto_tools_agent.py @@ -1,5 +1,9 @@ from typing import Literal, Type +from warnings import warn +from openai import AsyncOpenAI + +from sgr_deep_research.core.agent_definition import ExecutionConfig, LLMConfig, PromptsConfig from sgr_deep_research.core.agents.sgr_tools_agent import SGRToolCallingResearchAgent from sgr_deep_research.core.tools import BaseTool @@ -13,10 +17,23 @@ class SGRAutoToolCallingResearchAgent(SGRToolCallingResearchAgent): def __init__( self, task: str, + openai_client: AsyncOpenAI, + llm_config: LLMConfig, + prompts_config: PromptsConfig, + execution_config: ExecutionConfig, toolkit: list[Type[BaseTool]] | None = None, - max_clarifications: int = 3, - max_searches: int = 4, - max_iterations: int = 10, ): - super().__init__(task, toolkit, max_clarifications, max_searches, max_iterations) + super().__init__( + task=task, + openai_client=openai_client, + llm_config=llm_config, + prompts_config=prompts_config, + execution_config=execution_config, + toolkit=toolkit, + ) self.tool_choice: Literal["auto"] = "auto" + warn( + "SGRAutoToolCallingResearchAgent is deprecated and will be removed in the future. " + "This agent shows lower efficiency and stability based on our benchmarks.", + DeprecationWarning, + ) diff --git a/sgr_deep_research/core/agents/sgr_qwen3_thinking_agent.py b/sgr_deep_research/core/agents/sgr_qwen3_thinking_agent.py new file mode 100644 index 0000000..09dc23a --- /dev/null +++ b/sgr_deep_research/core/agents/sgr_qwen3_thinking_agent.py @@ -0,0 +1,239 @@ +"""SGR агент для qwen3-thinking моделей. + +Thinking модели семейства qwen3 не поддерживают Structured Output (SO), +но отлично работают через Function Calling (FC). Этот агент адаптирует +стандартную архитектуру SGR для работы с thinking моделями. +""" + +import inspect +from typing import Type + +from openai import AsyncOpenAI + +from sgr_deep_research.core.adapters.qwen3_thinking_adapter import extract_structured_response +from sgr_deep_research.core.agent_definition import ExecutionConfig, LLMConfig, PromptsConfig +from sgr_deep_research.core.base_agent import BaseAgent +from sgr_deep_research.core.tools import ( + BaseTool, + ClarificationTool, + CreateReportTool, + FinalAnswerTool, + NextStepToolsBuilder, + NextStepToolStub, + ReasoningTool, + WebSearchTool, +) +from sgr_deep_research.core.utils.pydantic_convert import pydantic_to_tools + + +def build_system_prompt_with_schema(schema_class, toolkit: list[Type[BaseTool]], base_prompt: str) -> str: + """Строит системный промпт со схемой для thinking моделей. + + Args: + schema_class: Pydantic-класс схемы NextStepTools + toolkit: Список доступных инструментов + base_prompt: Базовый системный промпт из конфигурации + + Returns: + Системный промпт с встроенной схемой + """ + try: + schema_code = inspect.getsource(schema_class) + except (OSError, TypeError): + # Если не удалось получить исходный код (динамически созданный класс) + schema_code = f"# Динамически созданная схема на основе {schema_class.__name__}" + + tool_descriptions = "\n".join([ + f"- {tool.tool_name}: {tool.description}" + for tool in toolkit + ]) + + return f"""{base_prompt} + +## Available Tools: +{tool_descriptions} + +## Response Schema for Thinking Models: +You must work strictly according to the following Pydantic schema: + +```python +{schema_code} +``` + +## CRITICAL RULES for Thinking Models: +- You may reason and think through the problem first in your internal monologue +- After reasoning, you MUST provide valid JSON matching the schema above +- Wrap your final JSON in ... tags for clarity +- All text fields in JSON must be concise and focused +- Example format: + +{{"reasoning_steps": ["step 1", "step 2"], "current_situation": "...", "function": {{"tool_name_discriminator": "tool_name", ...}}}} +""" + + +class SGRQwen3ThinkingAgent(BaseAgent): + """Agent for deep research tasks using SGR framework with qwen3-thinking models. + + Этот агент адаптирует стандартную архитектуру SGR для работы с thinking моделями, + используя Function Calling вместо Structured Output. + """ + + name: str = "sgr_qwen3_thinking_agent" + + def __init__( + self, + task: str, + openai_client: AsyncOpenAI, + llm_config: LLMConfig, + prompts_config: PromptsConfig, + execution_config: ExecutionConfig, + toolkit: list[Type[BaseTool]] | None = None, + ): + super().__init__( + task=task, + openai_client=openai_client, + llm_config=llm_config, + prompts_config=prompts_config, + execution_config=execution_config, + toolkit=toolkit, + ) + self.max_searches = execution_config.max_searches + # Удаляем ReasoningTool, так как используем свою схему рассуждений + if ReasoningTool in self.toolkit: + self.toolkit.remove(ReasoningTool) + + async def _prepare_tools(self) -> Type[NextStepToolStub]: + """Подготовка инструментов с учетом текущих ограничений контекста.""" + tools = set(self.toolkit) + if self._context.iteration >= self.max_iterations: + tools = { + CreateReportTool, + FinalAnswerTool, + } + if self._context.clarifications_used >= self.max_clarifications: + tools -= { + ClarificationTool, + } + if self._context.searches_used >= self.max_searches: + tools -= { + WebSearchTool, + } + return NextStepToolsBuilder.build_NextStepTools(list(tools)) + + async def _prepare_context(self) -> list[dict]: + """Подготовка контекста разговора с системным промптом для thinking моделей.""" + from sgr_deep_research.core.services.prompt_loader import PromptLoader + + next_step_schema = await self._prepare_tools() + base_prompt = PromptLoader.get_system_prompt(self.toolkit, self.prompts_config) + system_prompt = build_system_prompt_with_schema(next_step_schema, self.toolkit, base_prompt) + + return [ + {"role": "system", "content": system_prompt}, + *self.conversation, + ] + + async def _reasoning_phase(self) -> NextStepToolStub: + """Фаза рассуждения с использованием Function Calling для thinking моделей.""" + next_step_schema = await self._prepare_tools() + tool_name = "next_step" + + # Конвертируем Pydantic схему в OpenAI tools формат + tools = pydantic_to_tools( + next_step_schema, + name=tool_name, + description="Determine next reasoning step with adaptive planning" + ) + + # Вызываем модель с tools параметром + response = await self.openai_client.chat.completions.create( + model=self.llm_config.model, + messages=await self._prepare_context(), + tools=tools, + tool_choice={"type": "function", "function": {"name": tool_name}}, + max_tokens=self.llm_config.max_tokens, + temperature=self.llm_config.temperature, + ) + + message = response.choices[0].message + + # Извлекаем структурированный ответ с помощью адаптера + reasoning: NextStepToolStub = extract_structured_response( + response_message=message.model_dump(), + schema_class=next_step_schema, + tool_name=tool_name + ) + + self._log_reasoning(reasoning) + return reasoning + + async def _select_action_phase(self, reasoning: NextStepToolStub) -> BaseTool: + """Выбор действия на основе результата фазы рассуждения.""" + tool = reasoning.function + if not isinstance(tool, BaseTool): + raise ValueError("Selected tool is not a valid BaseTool instance") + + self.conversation.append( + { + "role": "assistant", + "content": reasoning.remaining_steps[0] if reasoning.remaining_steps else "Completing", + "tool_calls": [ + { + "type": "function", + "id": f"{self._context.iteration}-action", + "function": { + "name": tool.tool_name, + "arguments": tool.model_dump_json(), + }, + } + ], + } + ) + self.streaming_generator.add_tool_call( + f"{self._context.iteration}-action", tool.tool_name, tool.model_dump_json() + ) + return tool + + async def _action_phase(self, tool: BaseTool) -> str: + """Выполнение выбранного действия.""" + result = await tool(self._context) + self.conversation.append( + {"role": "tool", "content": result, "tool_call_id": f"{self._context.iteration}-action"} + ) + self.streaming_generator.add_chunk_from_str(f"{result}\n") + self._log_tool_execution(tool, result) + return result + + +if __name__ == "__main__": + import asyncio + import httpx + from sgr_deep_research.core.agent_config import GlobalConfig + + async def main(): + # Загружаем конфигурацию + config = GlobalConfig.from_yaml("config.yaml") + + # Создаем OpenAI клиент + client_kwargs = { + "base_url": config.llm.base_url, + "api_key": config.llm.api_key + } + if config.llm.proxy: + client_kwargs["http_client"] = httpx.AsyncClient(proxy=config.llm.proxy) + + openai_client = AsyncOpenAI(**client_kwargs) + + # Создаем агента + agent = SGRQwen3ThinkingAgent( + task="найди информацию о репозитории на гитхаб sgr-deep-research и ответь на вопрос, " + "какая основная концепция этого репозитория?", + openai_client=openai_client, + llm_config=config.llm, + prompts_config=config.prompts, + execution_config=config.execution, + toolkit=[WebSearchTool, CreateReportTool, ClarificationTool, FinalAnswerTool], + ) + await agent.execute() + + asyncio.run(main()) diff --git a/sgr_deep_research/core/agents/sgr_so_tools_agent.py b/sgr_deep_research/core/agents/sgr_so_tools_agent.py index 80ab108..b8ec61a 100644 --- a/sgr_deep_research/core/agents/sgr_so_tools_agent.py +++ b/sgr_deep_research/core/agents/sgr_so_tools_agent.py @@ -1,8 +1,12 @@ +from typing import Type +from warnings import warn + +from openai import AsyncOpenAI + +from sgr_deep_research.core.agent_definition import ExecutionConfig, LLMConfig, PromptsConfig from sgr_deep_research.core.agents.sgr_tools_agent import SGRToolCallingResearchAgent +from sgr_deep_research.core.base_tool import BaseTool from sgr_deep_research.core.tools import ReasoningTool -from sgr_deep_research.settings import get_config - -config = get_config() class SGRSOToolCallingResearchAgent(SGRToolCallingResearchAgent): @@ -11,12 +15,35 @@ class SGRSOToolCallingResearchAgent(SGRToolCallingResearchAgent): name: str = "sgr_so_tool_calling_agent" + def __init__( + self, + task: str, + openai_client: AsyncOpenAI, + llm_config: LLMConfig, + prompts_config: PromptsConfig, + execution_config: ExecutionConfig, + toolkit: list[Type[BaseTool]] | None = None, + ): + super().__init__( + task=task, + openai_client=openai_client, + llm_config=llm_config, + prompts_config=prompts_config, + execution_config=execution_config, + toolkit=toolkit, + ) + warn( + "SGRSOToolCallingResearchAgent is deprecated and will be removed in the future. " + "This agent shows lower efficiency and stability based on our benchmarks.", + DeprecationWarning, + ) + async def _reasoning_phase(self) -> ReasoningTool: async with self.openai_client.chat.completions.stream( - model=config.openai.model, + model=self.llm_config.model, messages=await self._prepare_context(), - max_tokens=config.openai.max_tokens, - temperature=config.openai.temperature, + max_tokens=self.llm_config.max_tokens, + temperature=self.llm_config.temperature, tools=await self._prepare_tools(), tool_choice={"type": "function", "function": {"name": ReasoningTool.tool_name}}, ) as stream: @@ -27,11 +54,11 @@ async def _reasoning_phase(self) -> ReasoningTool: (await stream.get_final_completion()).choices[0].message.tool_calls[0].function.parsed_arguments # ) async with self.openai_client.chat.completions.stream( - model=config.openai.model, + model=self.llm_config.model, response_format=ReasoningTool, messages=await self._prepare_context(), - max_tokens=config.openai.max_tokens, - temperature=config.openai.temperature, + max_tokens=self.llm_config.max_tokens, + temperature=self.llm_config.temperature, ) as stream: async for event in stream: if event.type == "chunk": diff --git a/sgr_deep_research/core/agents/sgr_tools_agent.py b/sgr_deep_research/core/agents/sgr_tools_agent.py index 4f0d39b..d9af978 100644 --- a/sgr_deep_research/core/agents/sgr_tools_agent.py +++ b/sgr_deep_research/core/agents/sgr_tools_agent.py @@ -1,8 +1,9 @@ from typing import Literal, Type -from openai import pydantic_function_tool +from openai import AsyncOpenAI, pydantic_function_tool from openai.types.chat import ChatCompletionFunctionToolParam +from sgr_deep_research.core.agent_definition import ExecutionConfig, LLMConfig, PromptsConfig from sgr_deep_research.core.agents.sgr_agent import SGRResearchAgent from sgr_deep_research.core.models import AgentStatesEnum from sgr_deep_research.core.tools import ( @@ -12,42 +13,33 @@ FinalAnswerTool, ReasoningTool, WebSearchTool, - research_agent_tools, - system_agent_tools, ) -from sgr_deep_research.services import MCP2ToolConverter -from sgr_deep_research.settings import get_config - -config = get_config() class SGRToolCallingResearchAgent(SGRResearchAgent): """Agent that uses OpenAI native function calling to select and execute - tools based on SGR like reasoning scheme.""" + tools based on SGR like a reasoning scheme.""" name: str = "sgr_tool_calling_agent" def __init__( self, task: str, + openai_client: AsyncOpenAI, + llm_config: LLMConfig, + prompts_config: PromptsConfig, + execution_config: ExecutionConfig, toolkit: list[Type[BaseTool]] | None = None, - max_clarifications: int = 3, - max_searches: int = 4, - max_iterations: int = 10, ): super().__init__( task=task, + openai_client=openai_client, + llm_config=llm_config, + prompts_config=prompts_config, + execution_config=execution_config, toolkit=toolkit, - max_clarifications=max_clarifications, - max_iterations=max_iterations, - max_searches=max_searches, ) - self.toolkit = [ - *system_agent_tools, - *research_agent_tools, - *MCP2ToolConverter().toolkit, - *(toolkit if toolkit else []), - ] + self.toolkit.append(ReasoningTool) self.tool_choice: Literal["required"] = "required" async def _prepare_tools(self) -> list[ChatCompletionFunctionToolParam]: @@ -71,10 +63,10 @@ async def _prepare_tools(self) -> list[ChatCompletionFunctionToolParam]: async def _reasoning_phase(self) -> ReasoningTool: async with self.openai_client.chat.completions.stream( - model=config.openai.model, + model=self.llm_config.model, messages=await self._prepare_context(), - max_tokens=config.openai.max_tokens, - temperature=config.openai.temperature, + max_tokens=self.llm_config.max_tokens, + temperature=self.llm_config.temperature, tools=await self._prepare_tools(), tool_choice={"type": "function", "function": {"name": ReasoningTool.tool_name}}, ) as stream: @@ -109,10 +101,10 @@ async def _reasoning_phase(self) -> ReasoningTool: async def _select_action_phase(self, reasoning: ReasoningTool) -> BaseTool: async with self.openai_client.chat.completions.stream( - model=config.openai.model, + model=self.llm_config.model, messages=await self._prepare_context(), - max_tokens=config.openai.max_tokens, - temperature=config.openai.temperature, + max_tokens=self.llm_config.max_tokens, + temperature=self.llm_config.temperature, tools=await self._prepare_tools(), tool_choice=self.tool_choice, ) as stream: diff --git a/sgr_deep_research/core/agents/tools_agent.py b/sgr_deep_research/core/agents/tools_agent.py index 2d80d66..95f54f2 100644 --- a/sgr_deep_research/core/agents/tools_agent.py +++ b/sgr_deep_research/core/agents/tools_agent.py @@ -1,23 +1,17 @@ from typing import Literal, Type -from openai import pydantic_function_tool +from openai import AsyncOpenAI, pydantic_function_tool from openai.types.chat import ChatCompletionFunctionToolParam -from sgr_deep_research.core.agents.base_agent import BaseAgent +from sgr_deep_research.core.agent_definition import ExecutionConfig, LLMConfig, PromptsConfig +from sgr_deep_research.core.base_agent import BaseAgent from sgr_deep_research.core.tools import ( BaseTool, ClarificationTool, CreateReportTool, FinalAnswerTool, - ReasoningTool, WebSearchTool, - research_agent_tools, - system_agent_tools, ) -from sgr_deep_research.services import MCP2ToolConverter -from sgr_deep_research.settings import get_config - -config = get_config() class ToolCallingResearchAgent(BaseAgent): @@ -29,27 +23,21 @@ class ToolCallingResearchAgent(BaseAgent): def __init__( self, task: str, + openai_client: AsyncOpenAI, + llm_config: LLMConfig, + prompts_config: PromptsConfig, + execution_config: ExecutionConfig, toolkit: list[Type[BaseTool]] | None = None, - max_clarifications: int = 3, - max_searches: int = 4, - max_iterations: int = 10, ): super().__init__( task=task, + openai_client=openai_client, + llm_config=llm_config, + prompts_config=prompts_config, + execution_config=execution_config, toolkit=toolkit, - max_clarifications=max_clarifications, - max_iterations=max_iterations, ) - - self.toolkit = [ - *system_agent_tools, - *research_agent_tools, - *MCP2ToolConverter().toolkit, - *(toolkit if toolkit else []), - ] - self.toolkit.remove(ReasoningTool) # LLM will do the reasoning internally - - self.max_searches = max_searches + self.max_searches = execution_config.max_searches self.tool_choice: Literal["required"] = "required" async def _prepare_tools(self) -> list[ChatCompletionFunctionToolParam]: @@ -76,10 +64,10 @@ async def _reasoning_phase(self) -> None: async def _select_action_phase(self, reasoning=None) -> BaseTool: async with self.openai_client.chat.completions.stream( - model=config.openai.model, + model=self.llm_config.model, messages=await self._prepare_context(), - max_tokens=config.openai.max_tokens, - temperature=config.openai.temperature, + max_tokens=self.llm_config.max_tokens, + temperature=self.llm_config.temperature, tools=await self._prepare_tools(), tool_choice=self.tool_choice, ) as stream: diff --git a/sgr_deep_research/core/agents/base_agent.py b/sgr_deep_research/core/base_agent.py similarity index 82% rename from sgr_deep_research/core/agents/base_agent.py rename to sgr_deep_research/core/base_agent.py index 821c906..5fba630 100644 --- a/sgr_deep_research/core/agents/base_agent.py +++ b/sgr_deep_research/core/base_agent.py @@ -6,26 +6,29 @@ from datetime import datetime from typing import Type -import httpx from openai import AsyncOpenAI from openai.types.chat import ChatCompletionFunctionToolParam +from sgr_deep_research.core.agent_definition import ExecutionConfig, LLMConfig, PromptsConfig from sgr_deep_research.core.models import AgentStatesEnum, ResearchContext -from sgr_deep_research.core.prompts import PromptLoader +from sgr_deep_research.core.services.prompt_loader import PromptLoader +from sgr_deep_research.core.services.registry import AgentRegistry from sgr_deep_research.core.stream import OpenAIStreamingGenerator from sgr_deep_research.core.tools import ( - # Base BaseTool, ClarificationTool, ReasoningTool, - system_agent_tools, ) -from sgr_deep_research.settings import get_config -config = get_config() +class AgentRegistryMixin: + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + if cls.__name__ not in ("BaseAgent",): + AgentRegistry.register(cls, name=cls.name) -class BaseAgent: + +class BaseAgent(AgentRegistryMixin): """Base class for agents.""" name: str = "base_agent" @@ -33,32 +36,36 @@ class BaseAgent: def __init__( self, task: str, + openai_client: AsyncOpenAI, + llm_config: LLMConfig, + prompts_config: PromptsConfig, + execution_config: ExecutionConfig, toolkit: list[Type[BaseTool]] | None = None, - max_iterations: int = 20, - max_clarifications: int = 3, + **kwargs: dict, ): self.id = f"{self.name}_{uuid.uuid4()}" self.logger = logging.getLogger(f"sgr_deep_research.agents.{self.id}") self.creation_time = datetime.now() self.task = task - self.toolkit = [*system_agent_tools, *(toolkit or [])] + self.toolkit = toolkit or [] self._context = ResearchContext() self.conversation = [] self.log = [] - self.max_iterations = max_iterations - self.max_clarifications = max_clarifications + self.max_iterations = execution_config.max_iterations + self.max_clarifications = execution_config.max_clarifications - client_kwargs = {"base_url": config.openai.base_url, "api_key": config.openai.api_key} - if config.openai.proxy.strip(): - client_kwargs["http_client"] = httpx.AsyncClient(proxy=config.openai.proxy) + self.openai_client = openai_client + self.llm_config = llm_config + self.prompts_config = prompts_config - self.openai_client = AsyncOpenAI(**client_kwargs) self.streaming_generator = OpenAIStreamingGenerator(model=self.id) async def provide_clarification(self, clarifications: str): """Receive clarification from external source (e.g. user input)""" - self.conversation.append({"role": "user", "content": PromptLoader.get_clarification_template(clarifications)}) + self.conversation.append( + {"role": "user", "content": PromptLoader.get_clarification_template(clarifications, self.prompts_config)} + ) self._context.clarifications_used += 1 self._context.clarification_received.set() self._context.state = AgentStatesEnum.RESEARCHING @@ -112,12 +119,14 @@ def _log_tool_execution(self, tool: BaseTool, result: str): ) def _save_agent_log(self): - logs_dir = config.execution.logs_dir + from sgr_deep_research.core.agent_config import GlobalConfig + + logs_dir = GlobalConfig().execution.logs_dir os.makedirs(logs_dir, exist_ok=True) filepath = os.path.join(logs_dir, f"{datetime.now().strftime('%Y%m%d-%H%M%S')}-{self.id}-log.json") agent_log = { "id": self.id, - "model_config": config.openai.model_dump(exclude={"api_key", "proxy"}), + "model_config": self.llm_config.model_dump(exclude={"api_key", "proxy"}), "task": self.task, "toolkit": [tool.tool_name for tool in self.toolkit], "log": self.log, @@ -128,7 +137,7 @@ def _save_agent_log(self): async def _prepare_context(self) -> list[dict]: """Prepare conversation context with system prompt.""" return [ - {"role": "system", "content": PromptLoader.get_system_prompt(self.toolkit)}, + {"role": "system", "content": PromptLoader.get_system_prompt(self.toolkit, self.prompts_config)}, *self.conversation, ] @@ -162,7 +171,7 @@ async def execute( [ { "role": "user", - "content": PromptLoader.get_initial_user_request(self.task), + "content": PromptLoader.get_initial_user_request(self.task, self.prompts_config), } ] ) diff --git a/sgr_deep_research/core/base_tool.py b/sgr_deep_research/core/base_tool.py index 6d3b857..b30dc97 100644 --- a/sgr_deep_research/core/base_tool.py +++ b/sgr_deep_research/core/base_tool.py @@ -7,17 +7,24 @@ from fastmcp import Client from pydantic import BaseModel -# from sgr_deep_research.core.models import AgentStatesEnum -from sgr_deep_research.settings import get_config +from sgr_deep_research.core.agent_config import GlobalConfig +from sgr_deep_research.core.services.registry import ToolRegistry if TYPE_CHECKING: from sgr_deep_research.core.models import ResearchContext -config = get_config() + logger = logging.getLogger(__name__) -class BaseTool(BaseModel): +class ToolRegistryMixin: + def __init_subclass__(cls, **kwargs) -> None: + super().__init_subclass__(**kwargs) + if cls.__name__ not in ("BaseTool", "MCPBaseTool"): + ToolRegistry.register(cls, name=cls.tool_name) + + +class BaseTool(BaseModel, ToolRegistryMixin): """Class to provide tool handling capabilities.""" tool_name: ClassVar[str] = None @@ -27,10 +34,10 @@ async def __call__(self, context: ResearchContext) -> str: """Result should be a string or dumped json.""" raise NotImplementedError("Execute method must be implemented by subclass") - def __init_subclass__(cls, **kwargs): - super().__init_subclass__(**kwargs) + def __init_subclass__(cls, **kwargs) -> None: cls.tool_name = cls.tool_name or cls.__name__.lower() cls.description = cls.description or cls.__doc__ or "" + super().__init_subclass__(**kwargs) class MCPBaseTool(BaseTool): @@ -39,12 +46,13 @@ class MCPBaseTool(BaseTool): _client: ClassVar[Client | None] = None async def __call__(self, _context) -> str: + config = GlobalConfig() payload = self.model_dump() try: async with self._client: result = await self._client.call_tool(self.tool_name, payload) return json.dumps([m.model_dump_json() for m in result.content], ensure_ascii=False)[ - : config.mcp.context_limit + : config.execution.mcp_context_limit ] except Exception as e: logger.error(f"Error processing MCP tool {self.tool_name}: {e}") diff --git a/sgr_deep_research/core/models.py b/sgr_deep_research/core/models.py index bb17878..7b5d844 100644 --- a/sgr_deep_research/core/models.py +++ b/sgr_deep_research/core/models.py @@ -62,7 +62,6 @@ class ResearchContext(BaseModel): default_factory=asyncio.Event, description="Event for clarification synchronization" ) - # ToDO: rename, my creativity finished now def agent_state(self) -> dict: return self.model_dump(exclude={"searches", "sources", "clarification_received"}) diff --git a/sgr_deep_research/core/next_step_tool.py b/sgr_deep_research/core/next_step_tool.py index 771bcb8..f8285ef 100644 --- a/sgr_deep_research/core/next_step_tool.py +++ b/sgr_deep_research/core/next_step_tool.py @@ -11,17 +11,13 @@ from sgr_deep_research.core.base_tool import BaseTool from sgr_deep_research.core.tools.reasoning_tool import ReasoningTool -# from sgr_deep_research.core.models import AgentStatesEnum -from sgr_deep_research.settings import get_config - -config = get_config() logger = logging.getLogger(__name__) T = TypeVar("T", bound=BaseTool) class NextStepToolStub(ReasoningTool, ABC): - """SGR Core - Determines next reasoning step with adaptive planning, choosing appropriate tool + """SGR Core - Determines the next reasoning step with adaptive planning, choosing appropriate tool (!) Stub class for correct autocomplete. Use NextStepToolsBuilder""" function: T = Field(description="Select the appropriate tool for the next step") @@ -38,12 +34,12 @@ def model_dump(self, *args, **kwargs): class NextStepToolsBuilder: - """SGR Core - Builder for NextStepTool with dynamic union tool function type on + """SGR Core - Builder for NextStepTool with a dynamic union tool function type on pydantic models level.""" @classmethod def _create_discriminant_tool(cls, tool_class: Type[T]) -> Type[BaseModel]: - """Create discriminant version of tool with tool_name as instance + """Create a discriminant version of tool with tool_name as an instance field.""" return create_model( # noqa @@ -57,7 +53,7 @@ def _create_tool_types_union(cls, tools_list: list[Type[T]]) -> Type: """Create discriminated union of tools.""" if len(tools_list) == 1: return cls._create_discriminant_tool(tools_list[0]) - # SGR inference struggles with choosing right schema otherwise + # SGR inference struggles with choosing the right schema otherwise discriminant_tools = [cls._create_discriminant_tool(tool) for tool in tools_list] union = reduce(operator.or_, discriminant_tools) return Annotated[union, Field()] diff --git a/sgr_deep_research/core/prompts.py b/sgr_deep_research/core/prompts.py deleted file mode 100644 index f93d8d6..0000000 --- a/sgr_deep_research/core/prompts.py +++ /dev/null @@ -1,49 +0,0 @@ -import os -from datetime import datetime -from functools import cache - -from sgr_deep_research.core.tools import BaseTool -from sgr_deep_research.settings import get_config - -config = get_config() - - -class PromptLoader: - @classmethod - @cache - def _load_prompt_file(cls, filename: str) -> str: - user_file_path = os.path.join(config.prompts.prompts_dir, filename) - lib_file_path = os.path.join(os.path.dirname(__file__), "..", config.prompts.prompts_dir, filename) - - for file_path in [user_file_path, lib_file_path]: - if os.path.exists(file_path): - try: - with open(file_path, encoding="utf-8") as f: - return f.read().strip() - except IOError as e: - raise IOError(f"Error reading prompt file {file_path}: {e}") from e - - raise FileNotFoundError(f"Prompt file not found: {user_file_path} or {lib_file_path}") - - @classmethod - def get_system_prompt(cls, available_tools: list[BaseTool]) -> str: - template = cls._load_prompt_file(config.prompts.system_prompt_file) - available_tools_str_list = [ - f"{i}. {tool.tool_name}: {tool.description}" for i, tool in enumerate(available_tools, start=1) - ] - try: - return template.format( - available_tools="\n".join(available_tools_str_list), - ) - except KeyError as e: - raise KeyError(f"Missing placeholder in system prompt template: {e}") from e - - @classmethod - def get_initial_user_request(cls, task: str) -> str: - template = cls._load_prompt_file("initial_user_request.txt") - return template.format(task=task, current_date=datetime.now().strftime("%Y-%m-%d %H:%M:%S")) - - @classmethod - def get_clarification_template(cls, clarifications: str) -> str: - template = cls._load_prompt_file("clarification_response.txt") - return template.format(clarifications=clarifications, current_date=datetime.now().strftime("%Y-%m-%d %H:%M:%S")) diff --git a/sgr_deep_research/prompts/__init__.py b/sgr_deep_research/core/prompts/__init__.py similarity index 100% rename from sgr_deep_research/prompts/__init__.py rename to sgr_deep_research/core/prompts/__init__.py diff --git a/sgr_deep_research/prompts/clarification_response.txt b/sgr_deep_research/core/prompts/clarification_response.txt similarity index 100% rename from sgr_deep_research/prompts/clarification_response.txt rename to sgr_deep_research/core/prompts/clarification_response.txt diff --git a/sgr_deep_research/prompts/initial_user_request.txt b/sgr_deep_research/core/prompts/initial_user_request.txt similarity index 100% rename from sgr_deep_research/prompts/initial_user_request.txt rename to sgr_deep_research/core/prompts/initial_user_request.txt diff --git a/sgr_deep_research/prompts/system_prompt.txt b/sgr_deep_research/core/prompts/system_prompt.txt similarity index 100% rename from sgr_deep_research/prompts/system_prompt.txt rename to sgr_deep_research/core/prompts/system_prompt.txt diff --git a/sgr_deep_research/core/services/__init__.py b/sgr_deep_research/core/services/__init__.py new file mode 100644 index 0000000..ceac54d --- /dev/null +++ b/sgr_deep_research/core/services/__init__.py @@ -0,0 +1,14 @@ +"""Services module for external integrations and business logic.""" + +from sgr_deep_research.core.services.mcp_service import MCP2ToolConverter +from sgr_deep_research.core.services.prompt_loader import PromptLoader +from sgr_deep_research.core.services.registry import AgentRegistry, ToolRegistry +from sgr_deep_research.core.services.tavily_search import TavilySearchService + +__all__ = [ + "TavilySearchService", + "MCP2ToolConverter", + "ToolRegistry", + "AgentRegistry", + "PromptLoader", +] diff --git a/sgr_deep_research/core/services/mcp_service.py b/sgr_deep_research/core/services/mcp_service.py new file mode 100644 index 0000000..e86536b --- /dev/null +++ b/sgr_deep_research/core/services/mcp_service.py @@ -0,0 +1,51 @@ +import logging +from typing import Type + +from fastmcp import Client +from fastmcp.mcp_config import MCPConfig +from jambo import SchemaConverter +from pydantic import create_model + +logger = logging.getLogger(__name__) + + +class MCP2ToolConverter: + @staticmethod + def _to_CamelCase(name: str) -> str: + return name.replace("_", " ").title().replace(" ", "") + + @classmethod + async def build_tools_from_mcp(cls, config: MCPConfig): + from sgr_deep_research.core import BaseTool, MCPBaseTool + + tools = [] + if not config.mcpServers: + return tools + + client: Client = Client(config) + async with client: + mcp_tools = await client.list_tools() + + for t in mcp_tools: + if not t.name or not t.inputSchema: + logger.error(f"Skipping tool due to missing name or input schema: {t}") + continue + + try: + t.inputSchema["title"] = cls._to_CamelCase(t.name) + PdModel = SchemaConverter.build(t.inputSchema) + except Exception as e: + logger.error(f"Error creating model {t.name} from schema: {t.inputSchema}: {e}") + continue + + ToolCls: Type[BaseTool] = create_model( + f"MCP{cls._to_CamelCase(t.name)}", __base__=(PdModel, MCPBaseTool), __doc__=t.description or "" + ) + ToolCls.tool_name = t.name + ToolCls.description = t.description or "" + ToolCls._client = client + tools.append(ToolCls) + logger.info(f"Built MCP Tool: {ToolCls.tool_name}") + + logger.info(f"Built {len(tools)} MCP tools.") + return tools diff --git a/sgr_deep_research/core/services/prompt_loader.py b/sgr_deep_research/core/services/prompt_loader.py new file mode 100644 index 0000000..e7346bb --- /dev/null +++ b/sgr_deep_research/core/services/prompt_loader.py @@ -0,0 +1,31 @@ +from datetime import datetime +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from sgr_deep_research.core.agent_definition import PromptsConfig + from sgr_deep_research.core.tools import BaseTool + + +class PromptLoader: + @classmethod + def get_system_prompt(cls, available_tools: list["BaseTool"], prompts_config: "PromptsConfig") -> str: + template = prompts_config.system_prompt + available_tools_str_list = [ + f"{i}. {tool.tool_name}: {tool.description}" for i, tool in enumerate(available_tools, start=1) + ] + try: + return template.format( + available_tools="\n".join(available_tools_str_list), + ) + except KeyError as e: + raise KeyError(f"Missing placeholder in system prompt template: {e}") from e + + @classmethod + def get_initial_user_request(cls, task: str, prompts_config: "PromptsConfig") -> str: + template = prompts_config.initial_user_request + return template.format(task=task, current_date=datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + + @classmethod + def get_clarification_template(cls, clarifications: str, prompts_config: "PromptsConfig") -> str: + template = prompts_config.clarification_response + return template.format(clarifications=clarifications, current_date=datetime.now().strftime("%Y-%m-%d %H:%M:%S")) diff --git a/sgr_deep_research/core/services/registry.py b/sgr_deep_research/core/services/registry.py new file mode 100644 index 0000000..00d5db7 --- /dev/null +++ b/sgr_deep_research/core/services/registry.py @@ -0,0 +1,124 @@ +import logging +from typing import TYPE_CHECKING, Generic, Tuple, TypeVar + +if TYPE_CHECKING: + from sgr_deep_research.core.base_agent import BaseAgent # noqa: F401 + from sgr_deep_research.core.base_tool import BaseTool # noqa: F401 + + +logger = logging.getLogger(__name__) + +T = TypeVar("T") + + +class Registry(Generic[T]): + """Generic registry for managing classes. + + Can be subclassed to create specific registries for different types. + Each subclass will have its own separate registry storage. + """ + + _items: dict[str, type[T]] = {} + + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + cls._items = {} + + def __init__(self): + raise TypeError(f"{self.__class__.__name__} is a static class and cannot be instantiated") + + @classmethod + def register(cls, item_class: type[T] | None = None, name: str | None = None) -> type[T]: + """Register an item class. + + Can be used as a decorator or called directly: + + As decorator: + @Registry.register + class MyClass: + pass + + As decorator with name: + @Registry.register(name="custom_name") + class MyClass: + pass + + Direct call: + Registry.register(MyClass) + Registry.register(MyClass, name="custom_name") + + Args: + item_class: Class to register (None when used as decorator) + name: Optional name to register the class under + + Returns: + The class itself (for decorator usage) or decorator function + """ + + def _register(cls_to_register: type[T]) -> type[T]: + """Internal registration function.""" + if cls_to_register.__name__ not in cls._items: + cls._items[cls_to_register.__name__.lower()] = cls_to_register + if name is not None: + cls._items[name.lower()] = cls_to_register + return cls_to_register + + # Used as decorator without arguments: @Registry.register + if item_class is not None: + return _register(item_class) + return _register + + @classmethod + def get(cls, name: str) -> type[T] | None: + """Get a class by name. + + Args: + name: Name of the class to retrieve + + Returns: + Class or None if not found + """ + return cls._items.get(name.lower()) + + @classmethod + def list_items(cls) -> list[type[T]]: + """Get all registered items. + + Returns: + List of classes + """ + return list(set(cls._items.values())) + + @classmethod + def resolve(cls, names: list[str]) -> Tuple[list[type[T]], list[str]]: + """Resolve names to classes. + + Args: + names: List of names to resolve + + Returns: + List of classes + """ + items = [] + missing = [] + for name in names: + if item_class := cls._items.get(name.lower()): + items.append(item_class) + else: + logger.warning(f"Item {name} not found in {cls.__name__}") + missing.append(name) + continue + return items, missing + + @classmethod + def clear(cls) -> None: + """Clear all registered items.""" + cls._items.clear() + + +class AgentRegistry(Registry["BaseAgent"]): + pass + + +class ToolRegistry(Registry["BaseTool"]): + pass diff --git a/sgr_deep_research/services/tavily_search.py b/sgr_deep_research/core/services/tavily_search.py similarity index 93% rename from sgr_deep_research/services/tavily_search.py rename to sgr_deep_research/core/services/tavily_search.py index 17960b4..fc591e6 100644 --- a/sgr_deep_research/services/tavily_search.py +++ b/sgr_deep_research/core/services/tavily_search.py @@ -2,16 +2,18 @@ from tavily import AsyncTavilyClient +from sgr_deep_research.core.agent_config import GlobalConfig from sgr_deep_research.core.models import SourceData -from sgr_deep_research.settings import get_config logger = logging.getLogger(__name__) class TavilySearchService: def __init__(self): - config = get_config() - self._client = AsyncTavilyClient(api_key=config.tavily.api_key, api_base_url=config.tavily.api_base_url) + config = GlobalConfig() + self._client = AsyncTavilyClient( + api_key=config.search.tavily_api_key, api_base_url=config.search.tavily_api_base_url + ) self._config = config @staticmethod diff --git a/sgr_deep_research/core/tools/clarification_tool.py b/sgr_deep_research/core/tools/clarification_tool.py index 838710e..5492159 100644 --- a/sgr_deep_research/core/tools/clarification_tool.py +++ b/sgr_deep_research/core/tools/clarification_tool.py @@ -29,7 +29,7 @@ class ClarificationTool(BaseTool): ) questions: list[str] = Field( description="3 specific clarifying questions (short and direct)", - min_length=3, + min_length=1, max_length=3, ) diff --git a/sgr_deep_research/core/tools/create_report_tool.py b/sgr_deep_research/core/tools/create_report_tool.py index 77c59a5..d3cb18f 100644 --- a/sgr_deep_research/core/tools/create_report_tool.py +++ b/sgr_deep_research/core/tools/create_report_tool.py @@ -8,15 +8,14 @@ from pydantic import Field +from sgr_deep_research.core.agent_config import GlobalConfig from sgr_deep_research.core.base_tool import BaseTool -from sgr_deep_research.settings import get_config if TYPE_CHECKING: from sgr_deep_research.core.models import ResearchContext logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) -config = get_config() class CreateReportTool(BaseTool): @@ -42,7 +41,7 @@ class CreateReportTool(BaseTool): async def __call__(self, context: ResearchContext) -> str: # Save report - reports_dir = config.execution.reports_dir + reports_dir = GlobalConfig().execution.reports_dir os.makedirs(reports_dir, exist_ok=True) timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") safe_title = "".join(c for c in self.title if c.isalnum() or c in (" ", "-", "_"))[:50] diff --git a/sgr_deep_research/core/tools/extract_page_content_tool.py b/sgr_deep_research/core/tools/extract_page_content_tool.py index 8cda452..6a2fb26 100644 --- a/sgr_deep_research/core/tools/extract_page_content_tool.py +++ b/sgr_deep_research/core/tools/extract_page_content_tool.py @@ -5,16 +5,15 @@ from pydantic import Field +from sgr_deep_research.core.agent_config import GlobalConfig from sgr_deep_research.core.base_tool import BaseTool -from sgr_deep_research.services.tavily_search import TavilySearchService -from sgr_deep_research.settings import get_config +from sgr_deep_research.core.services.tavily_search import TavilySearchService if TYPE_CHECKING: from sgr_deep_research.core.models import ResearchContext logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) -config = get_config() class ExtractPageContentTool(BaseTool): @@ -66,7 +65,7 @@ async def __call__(self, context: ResearchContext) -> str: if url in context.sources: source = context.sources[url] if source.full_content: - content_preview = source.full_content[: config.scraping.content_limit] + content_preview = source.full_content[: GlobalConfig().search.content_limit] formatted_result += ( f"{str(source)}\n\n**Full Content:**\n" f"{content_preview}\n\n" diff --git a/sgr_deep_research/core/tools/final_answer_tool.py b/sgr_deep_research/core/tools/final_answer_tool.py index 3be8bdd..161469d 100644 --- a/sgr_deep_research/core/tools/final_answer_tool.py +++ b/sgr_deep_research/core/tools/final_answer_tool.py @@ -7,14 +7,12 @@ from sgr_deep_research.core.base_tool import BaseTool from sgr_deep_research.core.models import AgentStatesEnum -from sgr_deep_research.settings import get_config if TYPE_CHECKING: from sgr_deep_research.core.models import ResearchContext logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) -config = get_config() class FinalAnswerTool(BaseTool): diff --git a/sgr_deep_research/core/tools/web_search_tool.py b/sgr_deep_research/core/tools/web_search_tool.py index 3bd2077..81d991b 100644 --- a/sgr_deep_research/core/tools/web_search_tool.py +++ b/sgr_deep_research/core/tools/web_search_tool.py @@ -6,17 +6,16 @@ from pydantic import Field +from sgr_deep_research.core.agent_config import GlobalConfig from sgr_deep_research.core.base_tool import BaseTool from sgr_deep_research.core.models import SearchResult -from sgr_deep_research.services.tavily_search import TavilySearchService -from sgr_deep_research.settings import get_config +from sgr_deep_research.core.services.tavily_search import TavilySearchService if TYPE_CHECKING: from sgr_deep_research.core.models import ResearchContext logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) -config = get_config() class WebSearchTool(BaseTool): @@ -47,7 +46,7 @@ class WebSearchTool(BaseTool): reasoning: str = Field(description="Why this search is needed and what to expect") query: str = Field(description="Search query in same language as user request") max_results: int = Field( - default_factory=lambda: min(config.search.max_results, 10), + default_factory=lambda: min(GlobalConfig().search.max_results, 10), description="Maximum results", ge=1, le=10, diff --git a/sgr_deep_research/core/utils/__init__.py b/sgr_deep_research/core/utils/__init__.py new file mode 100644 index 0000000..9d89709 --- /dev/null +++ b/sgr_deep_research/core/utils/__init__.py @@ -0,0 +1,11 @@ +"""Утилиты для работы с агентами SGR.""" + +from sgr_deep_research.core.utils.pydantic_convert import ( + pydantic_to_tools, + pydantic_to_json_schema, +) + +__all__ = [ + "pydantic_to_tools", + "pydantic_to_json_schema", +] diff --git a/sgr_deep_research/core/utils/pydantic_convert.py b/sgr_deep_research/core/utils/pydantic_convert.py new file mode 100644 index 0000000..fa91526 --- /dev/null +++ b/sgr_deep_research/core/utils/pydantic_convert.py @@ -0,0 +1,202 @@ +"""Утилиты для конвертации Pydantic-моделей в формат OpenAI function calling. + +Модуль предоставляет функции для автоматического преобразования Pydantic-моделей +в JSON Schema, совместимый с OpenAI Function Calling API. +""" + +from typing import Any, Dict, Type, get_origin, get_args, Union, Literal, Annotated + +from pydantic import BaseModel +from pydantic.fields import FieldInfo + + +def pydantic_to_tools( + model: Type[BaseModel], + name: str = None, + description: str = None +) -> list: + """Конвертирует Pydantic модель в формат OpenAI function calling tool. + + Args: + model: Pydantic модель для конвертации + name: Имя функции (по умолчанию используется имя класса) + description: Описание функции (по умолчанию используется docstring класса) + + Returns: + Список с одним элементом в формате OpenAI tools + """ + return [{ + "type": "function", + "function": { + "name": name or model.__name__, + "description": description or model.__doc__ or f"Function {model.__name__}", + "parameters": pydantic_to_json_schema(model) + } + }] + + +def pydantic_to_json_schema(model: Type[BaseModel]) -> Dict[str, Any]: + """Конвертирует Pydantic модель в JSON Schema для OpenAI. + + Args: + model: Pydantic модель + + Returns: + JSON Schema словарь + """ + properties = {} + required = [] + + for field_name, field_info in model.model_fields.items(): + field_schema = _field_to_json_schema(field_info, field_name) + properties[field_name] = field_schema + + # Проверим обязательность поля + if field_info.is_required(): + required.append(field_name) + + schema = { + "type": "object", + "properties": properties + } + + if required: + schema["required"] = required + + return schema + + +def _field_to_json_schema(field_info: FieldInfo, field_name: str) -> Dict[str, Any]: + """Конвертирует поле Pydantic в JSON Schema. + + Args: + field_info: Информация о поле + field_name: Имя поля + + Returns: + JSON Schema для поля + """ + field_type = field_info.annotation + schema = {} + + # Добавляем описание если есть + if field_info.description: + schema["description"] = field_info.description + + # Обрабатываем тип поля + schema.update(_type_to_json_schema(field_type)) + + # Извлекаем constraints из metadata (для Annotated типов) + if hasattr(field_info, 'metadata') and field_info.metadata: + for constraint in field_info.metadata: + if hasattr(constraint, 'ge'): # greater or equal + schema["minimum"] = constraint.ge + if hasattr(constraint, 'le'): # less or equal + schema["maximum"] = constraint.le + if hasattr(constraint, 'gt'): # greater than + schema["exclusiveMinimum"] = constraint.gt + if hasattr(constraint, 'lt'): # less than + schema["exclusiveMaximum"] = constraint.lt + if hasattr(constraint, 'min_length'): + schema["minLength"] = constraint.min_length + if hasattr(constraint, 'max_length'): + schema["maxLength"] = constraint.max_length + if hasattr(constraint, 'pattern'): + schema["pattern"] = constraint.pattern + + # Добавляем default если есть и поле не required + if field_info.default is not None and not field_info.is_required(): + schema["default"] = field_info.default + + # Добавляем примеры если есть + if hasattr(field_info, 'examples') and field_info.examples: + schema["examples"] = field_info.examples + + return schema + + +def _type_to_json_schema(python_type: Any) -> Dict[str, Any]: + """Конвертирует Python тип в JSON Schema тип. + + Args: + python_type: Python тип + + Returns: + JSON Schema тип + """ + origin = get_origin(python_type) + + # Обработка Literal["a", "b", "c"] + if origin is Literal: + args = get_args(python_type) + # Определяем тип из первого значения + if args: + first_val = args[0] + if isinstance(first_val, str): + base_type = "string" + elif isinstance(first_val, int): + base_type = "integer" + elif isinstance(first_val, bool): + base_type = "boolean" + else: + base_type = "string" + + return { + "type": base_type, + "enum": list(args) + } + return {"type": "string"} + + # Обработка Annotated[T, ...] + if origin is Annotated: + args = get_args(python_type) + # Первый аргумент - это сам тип + return _type_to_json_schema(args[0]) + + # Обработка Optional[T] и Union[T, None] + if origin is Union: + args = get_args(python_type) + # Фильтруем None из Union + non_none_args = [arg for arg in args if arg is not type(None)] + + if len(non_none_args) == 1: + # Optional[T] случай + return _type_to_json_schema(non_none_args[0]) + else: + # Множественный Union - используем anyOf + return { + "anyOf": [_type_to_json_schema(arg) for arg in non_none_args] + } + + # Обработка List[T] + if origin is list: + args = get_args(python_type) + item_schema = _type_to_json_schema(args[0]) if args else {"type": "string"} + return { + "type": "array", + "items": item_schema + } + + # Обработка Dict[K, V] + if origin is dict: + args = get_args(python_type) + value_schema = _type_to_json_schema(args[1]) if len(args) > 1 else {} + return { + "type": "object", + "additionalProperties": value_schema if value_schema else True + } + + # Обработка вложенных Pydantic моделей + if isinstance(python_type, type) and issubclass(python_type, BaseModel): + return pydantic_to_json_schema(python_type) + + # Базовые типы + type_mapping = { + str: {"type": "string"}, + int: {"type": "integer"}, + float: {"type": "number"}, + bool: {"type": "boolean"}, + type(None): {"type": "null"} + } + + return type_mapping.get(python_type, {"type": "string"}) diff --git a/sgr_deep_research/services/__init__.py b/sgr_deep_research/services/__init__.py deleted file mode 100644 index 4281aae..0000000 --- a/sgr_deep_research/services/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Services module for external integrations and business logic.""" - -from sgr_deep_research.services.mcp_service import MCP2ToolConverter -from sgr_deep_research.services.tavily_search import TavilySearchService - -__all__ = [ - "TavilySearchService", - "MCP2ToolConverter", -] diff --git a/sgr_deep_research/services/mcp_service.py b/sgr_deep_research/services/mcp_service.py deleted file mode 100644 index 7935e45..0000000 --- a/sgr_deep_research/services/mcp_service.py +++ /dev/null @@ -1,65 +0,0 @@ -import logging -from typing import Type - -from fastmcp import Client -from jambo import SchemaConverter -from pydantic import create_model - -from sgr_deep_research.core.tools import BaseTool, MCPBaseTool -from sgr_deep_research.settings import get_config - -logger = logging.getLogger(__name__) - - -class Singleton(type): - """Singleton metaclass.""" - - _instances = {} - - def __call__(cls, *args, **kwargs): - if cls not in cls._instances: - cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) - return cls._instances[cls] - - -class MCP2ToolConverter(metaclass=Singleton): - def __init__(self): - self.toolkit: list[Type[BaseTool]] = [] - if not get_config().mcp.transport_config: - logger.warning("No MCP configuration found. MCP2ToolConverter will not function properly.") - return - self.client: Client = Client(get_config().mcp.transport_config) - - def _to_CamelCase(self, name: str) -> str: - return name.replace("_", " ").title().replace(" ", "") - - async def build_tools_from_mcp(self): - if not get_config().mcp.transport_config: - logger.warning("No MCP configuration found. Nothing to build.") - return - - async with self.client: - mcp_tools = await self.client.list_tools() - - for t in mcp_tools: - if not t.name or not t.inputSchema: - logger.error(f"Skipping tool due to missing name or input schema: {t}") - continue - - try: - t.inputSchema["title"] = self._to_CamelCase(t.name) - PdModel = SchemaConverter.build(t.inputSchema) - except Exception as e: - logger.error(f"Error creating model {t.name} from schema: {t.inputSchema}: {e}") - continue - - ToolCls: Type[BaseTool] = create_model( - f"MCP{self._to_CamelCase(t.name)}", __base__=(PdModel, MCPBaseTool), __doc__=t.description or "" - ) - ToolCls.tool_name = t.name - ToolCls.description = t.description or "" - ToolCls._client = self.client - logger.info(f"Built MCP Tool: {ToolCls.tool_name}") - self.toolkit.append(ToolCls) - - logger.info(f"Built {len(self.toolkit)} MCP tools.") diff --git a/sgr_deep_research/settings.py b/sgr_deep_research/settings.py index 125fb43..5e7cbd8 100644 --- a/sgr_deep_research/settings.py +++ b/sgr_deep_research/settings.py @@ -1,125 +1,31 @@ -"""Application settings module using Pydantic and EnvYAML. - -Loads configuration from YAML file with environment variables support. -""" - import logging import logging.config -import os -from functools import cache from pathlib import Path import yaml -from envyaml import EnvYAML -from pydantic import BaseModel, Field - - -class OpenAIConfig(BaseModel): - """OpenAI API settings.""" - - api_key: str = Field(description="API key") - base_url: str = Field(default="https://api.openai.com/v1", description="Base URL") - model: str = Field(default="gpt-4o-mini", description="Model to use") - max_tokens: int = Field(default=8000, description="Maximum number of tokens") - temperature: float = Field(default=0.4, ge=0.0, le=1.0, description="Generation temperature") - proxy: str = Field(default="", description="Proxy URL (e.g., socks5://127.0.0.1:1081 or http://127.0.0.1:8080)") - - -class TavilyConfig(BaseModel): - """Tavily Search API settings.""" - - api_key: str = Field(description="Tavily API key") - api_base_url: str = Field(default="https://api.tavily.com", description="Tavily API base URL") - - -class SearchConfig(BaseModel): - """Search settings.""" - - max_results: int = Field(default=10, ge=1, description="Maximum number of search results") - - -class ScrapingConfig(BaseModel): - """Web scraping settings.""" - - enabled: bool = Field(default=False, description="Enable full text scraping") - max_pages: int = Field(default=5, gt=0, description="Maximum pages to scrape") - content_limit: int = Field(default=1500, gt=0, description="Content character limit per source") - - -class PromptsConfig(BaseModel): - """Prompts settings.""" - - prompts_dir: str = Field(default="prompts", description="Directory with prompts") - system_prompt_file: str = Field(default="system_prompt.txt", description="System prompt file") +from pydantic import Field +from pydantic_settings import BaseSettings, SettingsConfigDict +logger = logging.getLogger(__name__) -class ExecutionConfig(BaseModel): - """Application execution settings.""" - max_steps: int = Field(default=6, gt=0, description="Maximum number of execution steps") - reports_dir: str = Field(default="reports", description="Directory for saving reports") - logs_dir: str = Field(default="logs", description="Directory for saving bot logs") - - -class LoggingConfig(BaseModel): - """Logging configuration settings.""" - - config_file: str = Field(default="logging_config.yaml", description="Logging configuration file path") - - -class MCPConfig(BaseModel): - """MCP (Model Context Protocol) configuration settings.""" - - context_limit: int = Field(default=15000, gt=0, description="Maximum context length from MCP server response") - transport_config: dict = Field(default_factory=dict, description="MCP servers configuration") - - -class AppConfig(BaseModel): - """Main application configuration.""" - - openai: OpenAIConfig = Field(description="OpenAI settings") - tavily: TavilyConfig = Field(description="Tavily settings") - search: SearchConfig = Field(default_factory=SearchConfig, description="Search settings") - scraping: ScrapingConfig = Field(default_factory=ScrapingConfig, description="Scraping settings") - execution: ExecutionConfig = Field(default_factory=ExecutionConfig, description="Execution settings") - prompts: PromptsConfig = Field(default_factory=PromptsConfig, description="Prompts settings") - logging: LoggingConfig = Field(default_factory=LoggingConfig, description="Logging settings") - mcp: MCPConfig = Field(default_factory=MCPConfig, description="MCP settings") - - -class ServerConfig(BaseModel): - """Server configuration.""" +class ServerConfig(BaseSettings): + model_config = SettingsConfigDict(cli_parse_args=True) + logging_file: str = Field(default="logging_config.yaml", description="Logging configuration file path") + config_file: str = Field(default="config.yaml", description="sgr core configuration file path") + agents_file: str = Field(default="agents.yaml", description="Agents definitions file path") host: str = Field(default="0.0.0.0", description="Host to listen on") port: int = Field(default=8010, gt=0, le=65535, description="Port to listen on") -@cache -def get_config() -> AppConfig: - app_config_env: str = os.environ.get("APP_CONFIG", "config.yaml") - - # If path has no directory part, assume it's in current working directory - if os.path.basename(app_config_env) == app_config_env: - app_config_path = Path.cwd() / app_config_env - else: - app_config_path = Path(app_config_env) - - return AppConfig.model_validate(dict(EnvYAML(str(app_config_path)))) - - def setup_logging() -> None: """Setup logging configuration from YAML file.""" - logging_config_path = Path(get_config().logging.config_file) + logging_config_path = Path(ServerConfig().logging_file) if not logging_config_path.exists(): raise FileNotFoundError(f"Logging config file not found: {logging_config_path}") with open(logging_config_path, "r", encoding="utf-8") as f: logging_config = yaml.safe_load(f) - logs_dir = Path(get_config().execution.logs_dir) - logs_dir.mkdir(parents=True, exist_ok=True) - - reports_dir = Path(get_config().execution.reports_dir) - reports_dir.mkdir(parents=True, exist_ok=True) - logging.config.dictConfig(logging_config)