I added multiline input feature.

.env.template

@@ -6,19 +6,15 @@ WORK_DIR=
 ANTHROPIC_API_KEY=
 OPENAI_API_KEY=
 OPENROUTER_API_KEY=
-GOOGLE_API_KEY=
 OLLAMA_MODEL=
 LOCAL_MODEL_API_BASE=
 LOCAL_MODEL_NAME=
 
-# Optional, but highly recommended
-## For RAG tool of Researcher
-COHERE_API_KEY=
-
-# Optional
 ## For Manager agent
 TODOIST_API_KEY=
 TODOIST_PROJECT_ID=
+
+# Optional
 ## For automatic error check
 LOG_FILE=
 ## Frontend Feedback
@@ -28,8 +24,8 @@ EDIT_TRANSCRIPTION=
 ## Show planner intermediate reasoning
 SHOW_LOGIC_PLAN=
 
-# optional - LLM observability
-LANGCHAIN_TRACING_V2=true
+# Optional - LLM observability
+LANGCHAIN_TRACING_V2=
 LANGCHAIN_ENDPOINT="https://api.smith.langchain.com"
 LANGCHAIN_API_KEY=
 LANGCHAIN_PROJECT=

.github/PULL_REQUEST_TEMPLATE/template.md

@@ -0,0 +1,15 @@
+## Pull Request Template
+
+### Description
+Please provide a detailed description of the changes made in this pull request.
+
+### How to use
+If you created new functionality, please describe how it could be enabled and used.
+
+### Related Issue
+If this pull request addresses an existing issue, please reference it here (e.g., "Fixes #123").
+
+### Checklist
+- [ ] I have tested these changes locally.
+- [ ] I used docstrings on the begin of every function I created to describe it. Both humans and AI will have no problem to understand my code.
+- [ ] I'm making contribution to the `dev` branch. Direct contributions to `master` are not allowed. Don't worry, they will be merged to `master` on the nearest release.

docker-compose.yml

@@ -10,7 +10,6 @@ services:
       - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}
       - TODOIST_API_KEY=${TODOIST_API_KEY}
       - TODOIST_PROJECT_ID=${TODOIST_PROJECT_ID}
-      - COHERE_API_KEY=${COHERE_API_KEY}
       - LOG_FILE=${LOG_FILE:-}
     volumes:
       - .:/Clean_Coder
@@ -29,7 +28,6 @@ services:
       - WORK_DIR=/work_dir
       - OPENAI_API_KEY=${OPENAI_API_KEY}
       - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}
-      - COHERE_API_KEY=${COHERE_API_KEY}
       - LOG_FILE=${LOG_FILE:-}
     volumes:
       - .:/Clean_Coder

manager.py

@@ -16,11 +16,12 @@
 from langgraph.graph import StateGraph
 from src.tools.tools_project_manager import add_task, modify_task, finish_project_planning, reorder_tasks
 from src.tools.tools_coder_pipeline import prepare_list_dir_tool, prepare_see_file_tool, ask_human_tool
+from src.tools.rag.index_file_descriptions import prompt_index_project_files
 from src.utilities.manager_utils import actualize_tasks_list_and_progress_description, setup_todoist_project_if_needed, get_manager_messages
 from src.utilities.langgraph_common_functions import call_model, call_tool, multiple_tools_msg, no_tools_msg, empty_message_msg
 from src.utilities.start_project_functions import set_up_dot_clean_coder_dir
 from src.utilities.util_functions import join_paths
-from src.utilities.llms import init_llms
+from src.utilities.llms import init_llms_medium_intelligence
 from src.utilities.print_formatters import print_formatted
 import json
 import os
@@ -34,9 +35,13 @@ class Manager:
     def __init__(self):
         load_dotenv(find_dotenv())
         self.work_dir = os.getenv("WORK_DIR")
+        # initial project setup
         set_up_dot_clean_coder_dir(self.work_dir)
+        setup_todoist_project_if_needed()
+        prompt_index_project_files()
+
         self.tools = self.prepare_tools()
-        self.llms = init_llms(tools=self.tools, run_name="Manager")
+        self.llms = init_llms_medium_intelligence(tools=self.tools, run_name="Manager")
         self.manager = self.setup_workflow()
         self.saved_messages_path = join_paths(self.work_dir, ".clean_coder/manager_messages.json")
 
@@ -113,7 +118,6 @@ def setup_workflow(self):
 
     def run(self):
         print_formatted("😀 Hello! I'm Manager agent. Let's plan your project together!", color="green")
-        setup_todoist_project_if_needed()
 
         messages = get_manager_messages(self.saved_messages_path)
         inputs = {"messages": messages}

non_src/tests/manual_tests/planer_scenario_1.py

@@ -11,8 +11,8 @@
 
 load_dotenv(find_dotenv())
 
-folder_with_project_files = repo_directory.joinpath("non_src/tests/manual_tests/projects_files", "debugger_scenario_1_files")
-tmp_folder =  pathlib.Path(__file__).parent.resolve().joinpath("sandbox_work_dir")
+folder_with_project_files = repo_directory.joinpath("non_src/tests/manual_tests/projects_files", "planner_scenario_1_files")
+tmp_folder = pathlib.Path(__file__).parent.resolve().joinpath("sandbox_work_dir")
 setup_work_dir(manual_tests_folder=tmp_folder, test_files_dir=folder_with_project_files)
 
 task = "Make form wider, with green background. Improve styling."

requirements.txt

@@ -10,8 +10,6 @@ langchain-ollama==0.2.0
 playwright==1.47.0
 libsass==0.23.0
 openai==1.61.1
-cohere==5.10.0
-langchain-cohere==0.3.0
 chromadb==0.4.21
 todoist-api-python==2.1.7
 termcolor==2.4.0
@@ -30,4 +28,5 @@ pyright==1.1.390
 ruff==0.8.2
 httpx==0.27.2
 questionary==2.1.0
-pathspec==0.12.1
+pathspec==0.12.1
+numpy==1.26.4

single_task_coder.py

@@ -17,6 +17,7 @@
 from src.utilities.start_project_functions import set_up_dot_clean_coder_dir
 from src.utilities.util_functions import create_frontend_feedback_story
 from concurrent.futures import ThreadPoolExecutor
+from src.tools.rag.index_file_descriptions import prompt_index_project_files
 
 
 use_frontend_feedback = bool(os.getenv("FRONTEND_URL"))
@@ -54,8 +55,9 @@ def run_clean_coder_pipeline(task: str, work_dir: str, doc_harvest: bool = False
 
 if __name__ == "__main__":
     work_dir = os.getenv("WORK_DIR")
-    set_up_dot_clean_coder_dir(work_dir)
-    task = user_input("Provide task to be executed. ")
     if not work_dir:
         raise Exception("WORK_DIR variable not provided. Please add WORK_DIR to .env file")
-    run_clean_coder_pipeline(task, work_dir)
+    set_up_dot_clean_coder_dir(work_dir)
+    prompt_index_project_files()
+    task = user_input("Provide task to be executed. ")
+    run_clean_coder_pipeline(task, work_dir)

src/agents/debugger_agent.py

@@ -17,7 +17,7 @@
     read_coderrules,
     convert_images,
 )
-from src.utilities.llms import init_llms
+from src.utilities.llms import init_llms_medium_intelligence
 from src.utilities.langgraph_common_functions import (
     call_model, call_tool, ask_human, after_ask_human_condition, multiple_tools_msg, no_tools_msg, agent_looped_human_help,
 )
@@ -50,7 +50,7 @@ class Debugger():
     def __init__(self, files, work_dir, human_feedback, image_paths, playwright_code=None):
         self.work_dir = work_dir
         self.tools = prepare_tools(work_dir)
-        self.llms = init_llms(self.tools, "Debugger")
+        self.llms = init_llms_medium_intelligence(self.tools, "Debugger")
         self.system_message = SystemMessage(
             content=system_prompt_template.format(project_rules=read_coderrules())
         )

src/agents/executor_agent.py

@@ -7,7 +7,7 @@
 from langgraph.graph import StateGraph, END
 from dotenv import load_dotenv, find_dotenv
 from langchain.tools import tool
-from src.utilities.llms import init_llms
+from src.utilities.llms import init_llms_medium_intelligence
 from src.utilities.print_formatters import print_formatted, print_error
 from src.utilities.util_functions import (
     check_file_contents, exchange_file_contents, bad_tool_call_looped
@@ -43,7 +43,7 @@ class Executor():
     def __init__(self, files, work_dir):
         self.work_dir = work_dir
         self.tools = prepare_tools(work_dir)
-        self.llms = init_llms(self.tools, "Executor")
+        self.llms = init_llms_medium_intelligence(self.tools, "Executor")
         self.system_message = SystemMessage(
             content=system_prompt_template
         )

src/agents/frontend_feedback.py

@@ -1,6 +1,6 @@
 import os
 from langchain_core.messages import HumanMessage
-from src.utilities.llms import init_llms
+from src.utilities.llms import init_llms_medium_intelligence
 from src.utilities.start_work_functions import read_frontend_feedback_story
 import base64
 import textwrap
@@ -9,7 +9,7 @@
 from pydantic import BaseModel, Field
 
 
-llms = init_llms(run_name="Frontend Feedback")
+llms = init_llms_medium_intelligence(run_name="Frontend Feedback")
 
 llm = llms[0].with_fallbacks(llms[1:])
 

src/agents/planner_agent.py

@@ -7,15 +7,15 @@
 from src.utilities.langgraph_common_functions import after_ask_human_condition
 from src.utilities.user_input import user_input
 from src.utilities.graphics import LoadingAnimation
-from src.utilities.llms import init_llms_high_intelligence, init_llms_mini, init_llms
+from src.utilities.llms import init_llms_high_intelligence, init_llms_mini, init_llms_medium_intelligence
 import os
 
 
 load_dotenv(find_dotenv())
 
 llms_planners = init_llms_high_intelligence(run_name="Planner")
 llm_strong = llms_planners[0].with_fallbacks(llms_planners[1:])
-llms_middle_strength = init_llms(run_name="Plan finalizer")
+llms_middle_strength = init_llms_medium_intelligence(run_name="Plan finalizer")
 llm_middle_strength = llms_middle_strength[0].with_fallbacks(llms_middle_strength[1:])
 llms_controller = init_llms_mini(run_name="Plan Files Controller")
 llm_controller = llms_controller[0].with_fallbacks(llms_controller[1:])
@@ -67,7 +67,7 @@ def call_advanced_planner(state):
     logic_pseudocode = llm_strong.invoke(logic_planner_messages)
     print_formatted("\nIntermediate planning done. Finalizing plan...", color="light_magenta")
     if os.getenv("SHOW_LOGIC_PLAN"):
-        print(logic_pseudocode.content)
+        print_formatted(logic_pseudocode.content, color="light_yellow")
 
     state["plan_finalizer_messages"].append(HumanMessage(content=f"Logic pseudocode plan to follow:\n\n{logic_pseudocode.content}"))
     plan_finalizer_messages = state["plan_finalizer_messages"]

src/agents/researcher_agent.py

@@ -13,7 +13,7 @@
     call_model, call_tool, ask_human, after_ask_human_condition, no_tools_msg
 )
 from src.utilities.print_formatters import print_formatted
-from src.utilities.llms import init_llms_mini
+from src.utilities.llms import init_llms_medium_intelligence
 import os
 
 
@@ -27,7 +27,7 @@
 @tool
 def final_response_researcher(
         files_to_work_on: Annotated[List[str], "List of existing files to potentially introduce changes"],
-        reference_files: Annotated[List[str], "List of code files useful as a reference without images"],
+        reference_files: Annotated[List[str], "List of code files useful as a reference. There are files where similar task been implemented already."],
         template_images: Annotated[List[str], "List of template images"]):
     """That tool outputs list of files programmer will need to change and paths to graphical patterns if some.
     Use that tool only when you 100% sure you found all the files programmer will need to modify.
@@ -65,7 +65,7 @@ def __init__(self, work_dir):
         self.tools = [see_file, list_dir, final_response_researcher]
         if vdb_available():
             self.tools.append(retrieve_files_by_semantic_query)
-        self.llms = init_llms_mini(self.tools, "Researcher")
+        self.llms = init_llms_medium_intelligence(self.tools, "Researcher")
 
         # workflow definition
         researcher_workflow = StateGraph(AgentState)

src/prompts/binary_ranker.prompt

@@ -0,0 +1,8 @@
+You are a binary ranker. Evaluate if document can contain answer for a given question.
+Question: """{question}"""
+Filename: """{filename}"""
+Document: """{document}"""
+
+If the document is relevant to the question, output only '1'. 
+If it may be useful for programmer as contains similar code, but no relevant directly, also output only '1'. 
+If it is not relevant at all, output only '0'.

src/prompts/describe_file_chunks.prompt

@@ -0,0 +1,19 @@
+First, get known with info about project (may be useful, may be not):
+'''
+{coderrules}
+'''
+
+For the reference, you have code of whole file here:
+'''
+{file_code}
+'''
+
+Describe provided function/file_chunk in 4 sentences or less, focusing only on important information from integration point of view.
+Write what function/file chunk is responsible for.
+
+Go straight to the thing in description, without starting sentence.
+
+Here is file chunk to describe:
+'''
+{chunk_code}
+'''

src/prompts/planner_system.prompt

@@ -20,16 +20,13 @@ For additional context, here's the directory tree:
 
 Instructions:
 
-1. Plan the logic:
-   Outline the logic algorithm before proposing code changes.
-
-2. Draft a detailed modification plan:
+1. Draft a detailed modification plan:
    - Prioritize readability
    - Follow the DRY (Don't Repeat Yourself) principle
    - Use meaningful variable names
    - Write concise code
 
-3. Format code snippets in your plan properly:
+2. Format code snippets in your plan properly:
    In your code snippets, follow udiff format with filename we working on in the header. For each code modification, use the following structure:
 
    ```filename.extension
@@ -43,3 +40,4 @@ Instructions:
 Remember:
 - If you're unsure how to implement a given task, don't improvise. Simply state that you don't know. Assuming is not allowed - just tell "please provide me with more files" when needed.
 - When adjusting your plan based on user feedback, always provide a complete version of the plan, referenced to original file contents. Don't reference previous plan.
+- Previous plan proposition have not been implemented. Always reference your code changes to code files you have in the context, not to the previous plan proposition.

src/prompts/researcher_system.prompt

@@ -1,21 +1,26 @@
-As a curious filesystem researcher, examine files thoroughly, prioritizing comprehensive checks. 
-You checking a lot of different folders looking around for interesting files (hey, you are very curious!) before giving the final answer.
-The more folders/files you will check, the more they will pay you.
-When you discover significant dependencies from one file to another, ensure to inspect both.
-Important: you are can not modify any files! You are reasearching only, but modifications will introduce another guys. Do not execute the task, just prepare ground for it's execution.
-Your final selection should include files needed to be modified or needed as reference for a programmer 
-(for example to see how code in similar file implemented). 
-Avoid recommending unseen or non-existent files in final response.
-
-You need to point out all files programmer needed to see to execute the task and only that task. Task is:
+As a curious filesystem researcher, thoroughly inspect the files for a task by following these steps:
+
+1. Break down the task to identify which parts of the application are responsible for executing it. Identify the root of the problem.
+
+2. Search through various folders to find all necessary files needed to modify for completing the task. Explore numerous folders and files to maximize your understanding.
+
+3. When you find significant dependencies between files, examine both thoroughly.
+
+4. Remember, you are only researching. Do not modify any files; modifications will be handled by others. Just prepare the groundwork for task execution.
+
+5. Also identify files that need to be used as a reference for a programmer. Reference files should include examples where similar tasks have been solved or similar coding tools been used and can serve as code guidance.
+
+6. Only include files that exist and are necessary for the task. You must not provide information about files you haven’t seen or that don’t exist.
+
+Lastly, list all files the programmer needs to see to execute the task and only include those relevant to this specific task:
+
 '''
 {task}
 '''
 
-Here is some additional info about project:
+Here's some additional information about the project:
 '''
 {project_rules}
 '''
 
-First, provide reasoning about results of your previous action. Think what do you need to find now in order to accomplish the task.
-Next, call tool(s). You can use up to 3 tool cals simultaniousely to speed up research.
+First, think about what you need to find to accomplish the task based on past actions. Then, use up to 3 tools simultaneously to gather this information.