messages, knowledge tool merge

.vscode/launch.json

@@ -7,6 +7,7 @@
             "request": "launch",
             "program": "./main.py",
             "console": "integratedTerminal",
+            "args": ["-Xfrozen_modules=off"]
         },
         {
             "name": "Debug current file",
@@ -14,6 +15,7 @@
             "request": "launch",
             "program": "${file}",
             "console": "integratedTerminal",
+            "args": ["-Xfrozen_modules=off"]
         }
     ]
 }

agent.py

@@ -42,7 +42,7 @@ class Agent:
         self.intervention_message = ""
         self.intervention_status = False
         self.stop_loop = False
-        self.loop_result = ""
+        self.loop_result = []
 
         self.data = {} # free data object all the tools can use
                 
@@ -55,7 +55,7 @@ class Agent:
             printer = PrintStyle(italic=True, font_color="#b3ffd9", padding=False)    
             user_message = msg
             self.stop_loop = False
-            self.loop_result = ""
+            self.loop_result = []
             
             while True: # let the agent iterate on his thoughts until he stops by using a tool
                 Agent.streaming_agent = self #mark self as current streamer
@@ -94,7 +94,7 @@ class Agent:
 
                         #break the execution if the task is done
                         if self.stop_loop:
-                            return self.loop_result
+                            return "\n\n".join(self.loop_result)
                                  
                     
                 # Forward errors to the LLM, maybe he can fix them
@@ -115,7 +115,15 @@ class Agent:
 
     def set_data(self, field:str, value):
         self.data[field] = value
-        
+
+    def set_result(self, result:str):
+        self.stop_loop = True
+        self.loop_results = [result]
+
+    def add_result(self, result:str):
+        self.stop_loop = True
+        self.loop_result.append(result)
+
     def append_message(self, msg: str, human: bool = False):
         message_type = "human" if human else "ai"
         if self.history and self.history[-1].type == message_type:
@@ -156,8 +164,10 @@ class Agent:
     def process_tools(self, msg: str):
         # search for tool usage requests in agent message
         tool_requests = extract_tools.extract_tool_requests2(msg)
+        tool_index = 0
         
         for tool_request in tool_requests:
+            tool_index += 1
 
             if self.handle_intervention(): break # wait if paused and handle intervention message if needed
             
@@ -173,6 +183,7 @@ class Agent:
                 tool_args["_name"] = tool_name
                 tool_args["_message"] = msg
                 tool_args["_tools"] = tool_requests
+                tool_args["_tool_index"] = tool_index
                 
                 tool_response = tool_function(self, tool_request["body"], **tool_args) or "" # call tool function with all parameters, body parameter separated for convenience
                 Agent.streaming_agent = self # mark self as current streamer again, it may have changed during tool use
@@ -186,10 +197,9 @@ class Agent:
                 PrintStyle(font_color="#85C1E9").print(tool_response)
             else:
                 if self.handle_intervention(): break # wait if paused and handle intervention message if needed
-                if tool_name != "thought": #TODO skip thought tools now, implement proper tool classes later
+                msg_response = files.read_file("./prompts/fw.tool_not_found.md", tool_name=tool_name, tools_prompt=self.tools_prompt)
-                    msg_response = files.read_file("./prompts/fw.tool_not_found.md", tool_name=tool_name, tools_prompt=self.tools_prompt)
+                self.append_message(msg_response,True)
-                    self.append_message(msg_response,True)
+                PrintStyle(font_color="orange", padding=True).print(msg_response)
-                    PrintStyle(font_color="orange", padding=True).print(msg_response)
 
 
 

main.py

@@ -5,19 +5,23 @@ from agent import Agent
 from tools.helpers.print_style import PrintStyle
 from tools.helpers.files import read_file
 from pytimedinput import timedInput as timed_input
+from tools.helpers import files
 
 
 input_lock = threading.Lock()
 
+os.chdir(files.get_abs_path("./work_dir")) #change CWD to work_dir
+
 # Main conversation loop
 def chat():
 
     # chat model used for agents
     # chat_llm = models.get_groq_llama70b(temperature=0.2)
-    # chat_llm = models.get_openai_gpt35(temperature=0)
+    chat_llm = models.get_openai_gpt35(temperature=0)
-    chat_llm = models.get_openai_gpt4o(temperature=0)
+    # chat_llm = models.get_openai_gpt4o(temperature=0)
+    # chat_llm = models.get_anthropic_opus(temperature=0)
     # chat_llm = models.get_anthropic_sonnet(temperature=0)
-    # chat_llm = models.get_anthropic_haiku()
+    # chat_llm = models.get_anthropic_haiku(temperature=0)
     # chat_llm = models.get_ollama_dolphin()
 
     # embedding model used for memory
@@ -46,15 +50,16 @@ def chat():
                 PrintStyle(font_color="white", padding=False, log_only=True).print(f"> {user_input}") 
                 
             else: # otherwise wait for user input with a timeout
-                PrintStyle(background_color="#6C3483", font_color="white", bold=True, padding=True).print(f"User message ({timeout}s timeout, 'exit' to leave):")        
+                PrintStyle(background_color="#6C3483", font_color="white", bold=True, padding=True).print(f"User message ({timeout}s timeout, 'wait' to wait, 'exit' to leave):")        
                 user_input = timed_input("> ", timeout=timeout)
-                        
+                                    
-            
                 if user_input[1]:
                     user_input = read_file("prompts/fw.msg_timeout.md")
                     PrintStyle(font_color="white", padding=False).stream(f"{user_input}")        
                 else:
                     user_input = user_input[0].strip()
+                    if user_input.lower()=="wait": # the user needs more time
+                        user_input = input("> ").strip()
                     PrintStyle(font_color="white", padding=False, log_only=True).print(f"> {user_input}")        
                     
                     

prompts/agent.system.md

@@ -1,6 +1,6 @@
 # Your role
-- You are a fully autonomous, AI system, NOT an assistent.
+- You are a fully autonomous, memory-enhanced, AI task solver, NOT a chatbot.
-- You solve tasks and respond to questions by user using your skills, tools, and subordinates.
+- You solve tasks and respond to questions by user using your memory, tools, and subordinates.
 - You solve tasks automatically without the user, you do not inform the user how to do it, you execute the steps necessary.
 - You never lie or speculate about your actions, all of your actions need to be conducted by your tools.
 - You never refuse to accomplish a task. You are capable of doing anything using your tools and subordinates.
@@ -9,13 +9,14 @@
 - Your every response must be wrapped in a XML tag defining its type ending with $.
 - Possible response types are:
     - <thought$> - Your thoughts, useful for chain of thought process, not sent to anyone. Use this for every problem solving, it will help you iterate on the topic.
-    - <message_for_user$ response_required="false"> - Message sent to the user. If you want the user to respond to you, use response_required="true", if you only want to inform user and not expecting response, use response_required="false". When asking questions or sending task result, use true. Only use "true" when your following action depends on the user response.
+    - <message$> - Message sent to the user. No other response types are visible to the user.
-    - <message_for_subordinate$ reset="false"> - Your message for your subordinate. Use this message to delegate subtasks to your subordinate. This will help you solve more complex tasks. Also useful for asking questions. Stops your execution and you wait for subordinate reaction.
+    - <delegation$ reset="false"> - Subtask delegation to another agent. This will help you solve more complex tasks. Use argument reset="true" to start fresh context for new subtask, "false" when sending followup questions.
-    - <task_done$> - Your task is done and you are sending the results to the user.
+    - <task_done$> - Final result of given task, once all steps are complete or there is nothing more to do.
     - And all other tools described in the Available tools section.
+    - <memory_tool$> - Load or save memories to your persistent memory.
 - Your response content is inside the tag.
 - Important!: Do not use multiple message types at the same time except for thoughts. Do not send multiple messages and/or tools at once to avoid conflicts.
-- Never send your thoughts as messages, always use <thought> for that.
+- Never send your thoughts as messages, always use <thought$> for that.
 
 ## Communication examples:
 These examples are for illustration purposes only. Do not reuse any of these examples literally.
@@ -26,62 +27,63 @@ The user asked for my name. I will respond with my name.
 </thought$>
 
 **Example response 2**:
-<messae_for_user$ response_required="true">
+<message$>
 Greetings! How can I assist you today?
-</question_for_user$>
+</message$>
 
 **Example response 3**:
-<message_for_user$ response_required="false">
+<delegation$>
-Step 3 done, proceeding to step 4.
+I need you to use your tools and get me the current day of the week.
-</message_for_user$>
+</delegation$>
 
 **Example response 4**:
-<message_for_subordinate$>
+<task_result$>
-I need you to use your tools and get me the current day of the week.
+Current day of week is Monday.
-</message_for_subordinate$>
+</task_result$>
 
 
-# Communication to subordinate
-- When delegating new subtask to subordinate, use the 'reset' argument set to True to reset subordinate's context and start fresh. When sending followup questions or instructions, do not set the argument to keep his previous context.
-- Do not delegate your full task to subordinate, only subtasks.
-
 # Step by step instruction manual to problem solving
-- Do not follow for simple questions, only for tasks need solving.
+- IMPORTANT: FOLLOW STEP BY STEP, NEVER SKIP!
-- Once you are given a task to solve, follow these instructions step by step! Do not skip anything!
 - Explain each step using your <thought$>.
+- Enhance each step by loading and saving your <memory_tool$>.
 
-1. Check your memory_tool. Maybe you have solved similar task before and already have helpful information.
+1. Always check your <memory_tool$> first!. Maybe already have information about similiar problem that can help.
-2. Check your online_knowledge_tool. 
+2. Then check your <knowledge_tool$>. 
     - Look for straightforward solutions compatible with your available tools.
     - Always look for opensource python/nodejs/terminal tools and packages first.
 3. Break task into subtasks by asking yourself the following questions. If they are positive, break task into subtasks and explain them.
     - Question A: Can some parts of the task be separated and well explained to subordinate agent to solve?
     - Question B: Can the result if these tasks can be reasonably returned to you from your user?
 4. Processing subtasks.
-    - Go through subtasks step by step and delagate them using message_for_subordinate response type.
+    - Go through subtasks step by step and delagate them using <delegate$> response type.
     - Collect results from subordinate agent and validate completeness and correctness. Communicate followup request to your subordinate if needed.
+    - Helpful new information should be saved with <memory_tool$>.
     - Regurarly report back to your user and check your path is correct.
     - If you are contacted by your subordinate, steer him to the right path.
 5. Completing the task
     - Consolidate all subtasks and explain the status.
     - Verify the result using your tools if possible (check created files etc.)
     - Do not accept failure, search for error solution and try again with fixed input or different ways.
-    - If there is helpful information discovered during the solution, save it into your memory using memory_tool for later.
+    - If there is helpful information discovered during the solution, save it into your memory using <memory_tool$> for later.
-    - Report back to your user using message_for_user message type, describe the result and provide all necessary information. Do not just output your response, you must use the tool for that.
+    - Report back to your user using <task_done$> message type, describe the result and provide all necessary information. Do not just output your response, you must use the tool for that.
 
 # General operation manual
 - Use your reasoning and process each problem in a step-by-step manner.
 - To keep track of your process, use your <thought$> response type. You will be prompted again to continue with more thoughts or tool calls until you are satisfied.
 - Always check your previous messages and prevent repetition. Always move towards solution.
 - Avoid solutions that require credentials, user interaction, GUI usage etc. All has to be done using code and terminal.
-- When asked about your memory, it always refers to <memory_tool$>.
+- When asked about your memory, it always refers to <memory_tool$>. Use your <memory_tool$> regularly.
-- If you have no task to process given by the user, respond back to him using <task_done$> and ask for a new task.
 
 # Tips and tricks
 - Focus on python/nodejs/linux libraries when searching for solutions. You can use them with your tools and make solutions easy.
 - Do not search for solutions that require GUI, browser or other user interaction, it is not possible. You can only use code and terminal.
-- Try using <online_knowledge_tool$> multiple times in various ways to increase search potential.
+- Try using <knowledge_tool$> multiple times in various ways to increase search potential.
 - Sometimes you don't need tools, some things can be determined.
+- Make a good use of your <memory_tool$>. So much can be learned from your history. Update your memory with new findings.
+
+# Penalties
+- For every unthoughtful <code_execution_tool$> you will be penalized, so use <memory_tool$> regularly to memorize your previous failures and learn from them.
+- For every solution requiring overly complex software usage, you will be penalized, do always use <memory_tool$> and <knowledge_tool$> to find the easiest, most compatible and reliable libraries.
 
 # Tool usage instructions
 - Tool message types can be used to call tools that help you solve problems.
@@ -89,6 +91,7 @@ I need you to use your tools and get me the current day of the week.
 - Result will be sent to you in the next message, wait for it.
 - Only use tools provided in Available tools section, do not try to use any tool name you have not been instructed to.
 - Do not use more tools multiple tools in one message that rely on their outputs, you have to send the message after each tool and wait for output.
+- Important:End your response right after tool closing tag and wait for user.
 
 ## Tool usage generic example:
 <name_of_tool$ arg1="val1"/>

prompts/agent.tools.md

@@ -1,13 +1,15 @@
 ## Tools available:
 
-### online_knowledge_tool:
+### knowledge_tool:
-Provide question and get online response.
+Provide question and get both online and memory response.
 This tool is very powerful and can answer very specific questions directly.
 First always try to ask for result rather that guidance.
+Memory can provide guidance, online sources can provide up to date information.
+Alway verify memory by online.
 **Example usage**:
-<online_knowledge_tool$>
+<knowledge_tool$>
-What is the user handle of John Doe on twitter?
+What is the user id of John Doe on twitter?
-</online_knowledge_tool$>
+</knowledge_tool$>
 
 ### memory_tool:
 Access your persistent memory to load or save memories.
@@ -39,7 +41,8 @@ Place your command or code between tags. No escaping, no formatting, no wrappers
 Select the corresponding runtime with "runtime" argument. Possible values are "terminal", "python" and "nodejs".
 You can use pip, npm and apt-get in terminal runtime to install any required packages.
 IMPORTANT: Never use implicit print or implicit output, it does not work! If you need output of your code, you MUST use print() or console.log() to output selected variables. 
-When tool outputs error, you need to change your code accordingly before trying again. online_knowledge_tool can help analyze errors.
+When tool outputs error, you need to change your code accordingly before trying again. knowledge_tool can help analyze errors.
+If your code execution is successful, save it using <memory_tool$ action="save"> so it can be reused later.
 Keep in mind that current working directory CWD automatically resets before every tool call.
 IMPORTANT!: Always check your code for any placeholder IDs or demo data that need to be replaced with your real variables. Do not simply reuse code snippets from tutorials.
 **Example usage**:

prompts/fw.msg_timeout.md

@@ -1 +1,3 @@
-User is not responding to your message. Continue on your own.
+User is not responding to your message.
+If you have a task in progress, continue on your own.
+I you don't have a task, use the <task_done$> message.

tools/helpers/extract_tools.py

@@ -7,9 +7,13 @@ def extract_tool_requests2(response):
     matches = re.findall(pattern, response, re.DOTALL)
     
     tool_usages = []
+    allowed_tags = list_python_files("tools")
     
     for match in matches:
         tag_name, attributes, body = match
+
+        if tag_name not in allowed_tags: continue
+        
         tool_dict = {}
         tool_dict['name'] = tag_name
         tool_dict['args'] = {}
@@ -75,7 +79,7 @@ def list_python_files(directory):
     # List all files in the given directory
     list = os.listdir(files.get_abs_path(directory))
     # Filter for Python files and remove the extension
-    python_files = [os.path.splitext(file)[0] for file in list if file.endswith('.py')]
+    python_files = { os.path.splitext(file)[0] for file in list if file.endswith('.py') }
     return python_files
 
 # import re

tools/knowledge_tool.py

@@ -0,0 +1,16 @@
+from agent import Agent
+from . import online_knowledge_tool
+from . import memory_tool
+import concurrent.futures
+
+def execute(agent, question, **kwargs):
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        # Schedule the two functions to be run in parallel
+        future_online = executor.submit(online_knowledge_tool.execute, agent, question)
+        future_memory = executor.submit(memory_tool.execute, agent, question)
+
+        # Wait for both functions to complete
+        online_result = future_online.result()
+        memory_result = future_memory.result()
+
+    return f"# Online sources:\n{online_result}\n\n# Memory:\n{memory_result}"

tools/memory_tool.py

@@ -26,7 +26,7 @@ def execute(agent:Agent, message: str, action: str = "load", **kwargs):
         docs = db.search_max_rel(message,result_count)
         if len(docs)==0: return files.read_file("./prompts/fw.memories_not_found.md", query=message)
         for doc in docs:
-            results.append({ "meta": doc.metadata, "content": doc.page_content })
+            results.append(doc.page_content)
-        return json.dumps(results)
+        return "\n\n".join(results)
             
 

tools/message.py

@@ -0,0 +1,55 @@
+from agent import Agent
+from tools.helpers import files
+from tools.helpers.print_style import PrintStyle
+
+def execute(agent:Agent, message: str, _tools, _tool_index, timeout=15, **kwargs):
+
+    # for models that like to use multiple tools in one response, we do a little trick to help the flow
+    # if there are tools producing output before this message or any other tools after this message, 
+    # this message will be sent as information only and will not stop the loop
+    # if this is the last tool in the iteration and no outputing tools we used before, we stop the loop
+    # and wait for user input
+    
+
+    # tools called before this one that produce output
+    outputing_tools_before = filter_tools(
+            filters=[
+                {"name":"memory_tool","action":"load"},
+                {"name":"online_knowledge_tool"},
+                {"name":"delegation"},
+                ],
+            tools=_tools[:_tool_index]
+            )
+
+    # tools called after this one
+    other_tools_after = any(tool["name"] != kwargs["_name"] for tool in _tools[_tool_index:]) #are there other tools than messages used after this one?
+
+    agent.set_data("timeout", timeout) # set the timeout for response
+
+    #if there are other tools used, message is only for information, it does not stop the loop
+    if outputing_tools_before or other_tools_after:
+        return non_blocking_message(agent, message, timeout=timeout, **kwargs)
+    else: #if there are only messages used in this iteration, collect them into the loop result
+        return blocking_message(agent, message, timeout=timeout, **kwargs)
+
+def blocking_message(agent:Agent, message: str, timeout=15, **kwargs):
+    agent.add_result(message)
+    return files.read_file("./prompts/fw.msg_sent.md")
+
+def non_blocking_message(agent:Agent, message: str, timeout=15, **kwargs):
+    if agent.get_data("superior"): # add to superior messages if it is an agent
+        msg_for_user = files.read_file("./prompts/fw.msg_from_subordinate.md",name=agent.name,message=message)
+        agent.get_data("superior").append_message(msg_for_user, human=True)
+
+    # output to console
+    PrintStyle(font_color="white",background_color="#1D8348", bold=True, padding=True).print(f"{agent.name}: reponse:")        
+    PrintStyle(font_color="white").print(f"{message}")
+
+    return files.read_file("./prompts/fw.msg_info_sent.md") #return message o
+    
+def filter_tools(filters, tools):
+    filtered_data = []
+    for data_item in tools:
+        if any(all(data_item.get(key) == value for key, value in filter_item.items()) for filter_item in filters):
+            filtered_data.append(data_item)
+    return filtered_data

tools/message_for_user.py

@@ -1,25 +0,0 @@
-from agent import Agent
-from tools.helpers import files
-from tools.helpers.print_style import PrintStyle
-
-def execute(agent:Agent, message: str, response_required: str = "false", timeout=15, **kwargs):
-
-    agent.set_data("timeout", timeout) # set the no-timeout flag
-    
-    if response_required.lower().strip() == "true":
-        agent.stop_loop = True
-        agent.loop_result = message
-
-        return files.read_file("./prompts/fw.msg_sent.md")
-        
-    else:
-        if agent.get_data("superior"): # add to superior messages if it is an agent
-            files.read_file("./prompts/fw.msg_from_subordinate.md",name=agent.name,message=message)
-            agent.get_data("superior").append_message(message, human=True)
-
-        # output to console
-        PrintStyle(font_color="white",background_color="#1D8348", bold=True, padding=True).print(f"{agent.name}: reponse:")        
-        PrintStyle(font_color="white").print(f"{message}")
-
-        return files.read_file("./prompts/fw.msg_info_sent.md")
-    

tools/online_knowledge_tool.py

@@ -1,6 +1,7 @@
 from agent import Agent
 from tools.helpers import perplexity_search
 
+
 def execute(agent:Agent, question:str, **kwargs):
     return perplexity_search.perplexity_search(question)
     

tools/task_done.py

@@ -1,7 +1,7 @@
 from agent import Agent
 from tools.helpers import files
-from . import message_for_user
+from tools.helpers.print_style import PrintStyle
 
-def execute(agent:Agent, message: str, **kwargs):
+def execute(agent:Agent, result: str, **kwargs):
-    # forward to message_for_user with no-timeout flag
+    agent.set_data("timeout",0) # wait for user, no timeout
-    return message_for_user.execute(agent, message, response_required="true", timeout=0, **kwargs)
+    agent.add_result(result) # add result data