From 6ed0d726e0068106f75330376127aedf68deae2c Mon Sep 17 00:00:00 2001 From: Kevin Backhouse Date: Tue, 2 Dec 2025 17:35:32 +0000 Subject: [PATCH 01/10] Use platformdirs.user_log_dir to create a log directory. --- .devcontainer/post-create.sh | 7 ------- .dockerignore | 1 - .gitignore | 5 ----- README.md | 1 - docker/run.sh | 3 --- src/seclab_taskflow_agent/__main__.py | 5 ++--- .../mcp_servers/codeql/client.py | 4 ++-- .../mcp_servers/codeql/mcp_server.py | 15 ++++++++------- .../mcp_servers/echo/echo.py | 8 +++++--- .../mcp_servers/logbook/logbook.py | 13 +++++++------ .../mcp_servers/memcache/memcache.py | 15 ++++++++------- src/seclab_taskflow_agent/path_utils.py | 8 ++++++++ src/seclab_taskflow_agent/render_utils.py | 12 ++++++++++-- 13 files changed, 50 insertions(+), 47 deletions(-) diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh index 1519242..eb4cdb5 100644 --- a/.devcontainer/post-create.sh +++ b/.devcontainer/post-create.sh @@ -17,17 +17,10 @@ hatch build # Install this package from local directory. pip install -e . -# Create logs directory if it doesn't exist -mkdir -p logs - -# Create optional data directories -mkdir -p data - # Create .env file if it doesn't exist if [ ! -f .env ]; then echo "📝 Creating .env template..." echo "# Optional: CodeQL database base path" >> .env - echo "CODEQL_DBS_BASE_PATH=$(realpath data)" >> .env echo "⚠️ Please configure the environment or your .env file with required tokens!" fi diff --git a/.dockerignore b/.dockerignore index a815e83..a608da8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,5 @@ __pycache__ docker/ -logs/**/* .env .venv .direnv diff --git a/.gitignore b/.gitignore index f8b7068..bc7983e 100644 --- a/.gitignore +++ b/.gitignore @@ -13,9 +13,6 @@ __pycache__/ # C extensions *.so -# Logs directory -logs/ - # Distribution / packaging .Python build/ @@ -186,5 +183,3 @@ config.yaml #database *.db -#logs -logs/ \ No newline at end of file diff --git a/README.md b/README.md index 39f9c76..b1a3f51 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,6 @@ if [ ! -f ".env" ]; then fi docker run \ - --volume "$PWD"/logs:/app/logs \ --mount type=bind,src="$PWD"/.env,dst=/app/.env,ro \ ${MY_DATA:+--mount type=bind,src=$MY_DATA,dst=/app/my_data} \ ${MY_MCP_SERVERS:+--mount type=bind,src=$MY_MCP_SERVERS,dst=/app/my_mcp_servers,ro} \ diff --git a/docker/run.sh b/docker/run.sh index 6903bfd..db880d7 100755 --- a/docker/run.sh +++ b/docker/run.sh @@ -16,12 +16,9 @@ # sudo -E ../docker/run.sh -p seclab_taskflow_agent.personalities.assistant 'explain modems to me please' touch -a .env -mkdir -p logs -mkdir -p data docker run -i \ --mount type=bind,src="$PWD",dst=/app \ - -e DATA_DIR=/app/data \ -e GITHUB_PERSONAL_ACCESS_TOKEN="$GITHUB_PERSONAL_ACCESS_TOKEN" \ -e COPILOT_TOKEN="$COPILOT_TOKEN" \ "ghcr.io/githubsecuritylab/seclab-taskflow-agent" "$@" diff --git a/src/seclab_taskflow_agent/__main__.py b/src/seclab_taskflow_agent/__main__.py index 6ea873d..ca2a554 100644 --- a/src/seclab_taskflow_agent/__main__.py +++ b/src/seclab_taskflow_agent/__main__.py @@ -33,15 +33,14 @@ from .agent import TaskAgent from .capi import list_tool_call_models from .available_tools import AvailableTools +from .path_utils import log_file_name load_dotenv(find_dotenv(usecwd=True)) # only model output or help message should go to stdout, everything else goes to log logging.getLogger('').setLevel(logging.NOTSET) -log_dir = pathlib.Path("logs") -log_dir.mkdir(parents=True, exist_ok=True) log_file_handler = RotatingFileHandler( - log_dir.joinpath('task_agent.log'), + log_file_name('task_agent.log'), maxBytes=1024*1024*10, backupCount=10) log_file_handler.setLevel(os.getenv('TASK_AGENT_LOGLEVEL', default='DEBUG')) diff --git a/src/seclab_taskflow_agent/mcp_servers/codeql/client.py b/src/seclab_taskflow_agent/mcp_servers/codeql/client.py index 5ef775d..29a8637 100644 --- a/src/seclab_taskflow_agent/mcp_servers/codeql/client.py +++ b/src/seclab_taskflow_agent/mcp_servers/codeql/client.py @@ -12,6 +12,7 @@ import os import zipfile import yaml +from .path_utils import log_file_name # this is a local fork of https://github.com/riga/jsonrpyc modified for our purposes from . import jsonrpyc @@ -45,8 +46,7 @@ def __init__(self, log_stderr=False): self.server_options = server_options.copy() if log_stderr: - os.makedirs("logs", exist_ok=True) - self.stderr_log = f"logs/codeql_stderr_log.log" + self.stderr_log = log_file_name(codeql_stderr_log.log) self.server_options.append("--log-to-stderr") else: self.stderr_log = os.devnull diff --git a/src/seclab_taskflow_agent/mcp_servers/codeql/mcp_server.py b/src/seclab_taskflow_agent/mcp_servers/codeql/mcp_server.py index 331ac67..b55e7b9 100644 --- a/src/seclab_taskflow_agent/mcp_servers/codeql/mcp_server.py +++ b/src/seclab_taskflow_agent/mcp_servers/codeql/mcp_server.py @@ -2,12 +2,6 @@ # SPDX-License-Identifier: MIT import logging -logging.basicConfig( - level=logging.DEBUG, - format='%(asctime)s - %(levelname)s - %(message)s', - filename='logs/mcp_codeql.log', - filemode='a' -) from .client import run_query, file_from_uri, list_src_files, _debug_log, search_in_src_archive from pydantic import Field #from mcp.server.fastmcp import FastMCP, Context @@ -20,7 +14,14 @@ import re from urllib.parse import urlparse, unquote import zipfile -from seclab_taskflow_agent.path_utils import mcp_data_dir +from seclab_taskflow_agent.path_utils import mcp_data_dir, log_file_name + +logging.basicConfig( + level=logging.DEBUG, + format='%(asctime)s - %(levelname)s - %(message)s', + filename=log_file_name('mcp_codeql.log'), + filemode='a' +) mcp = FastMCP("CodeQL") diff --git a/src/seclab_taskflow_agent/mcp_servers/echo/echo.py b/src/seclab_taskflow_agent/mcp_servers/echo/echo.py index 5667137..c202f8f 100644 --- a/src/seclab_taskflow_agent/mcp_servers/echo/echo.py +++ b/src/seclab_taskflow_agent/mcp_servers/echo/echo.py @@ -2,14 +2,16 @@ # SPDX-License-Identifier: MIT import logging +#from mcp.server.fastmcp import FastMCP +from fastmcp import FastMCP # move to FastMCP 2.0 +from .path_utils import log_file_name + logging.basicConfig( level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s', - filename='logs/mcp_echo.log', + filename=log_file_name('mcp_echo.log'), filemode='a' ) -#from mcp.server.fastmcp import FastMCP -from fastmcp import FastMCP # move to FastMCP 2.0 mcp = FastMCP("Echo") diff --git a/src/seclab_taskflow_agent/mcp_servers/logbook/logbook.py b/src/seclab_taskflow_agent/mcp_servers/logbook/logbook.py index 8c8bec0..5115ba4 100644 --- a/src/seclab_taskflow_agent/mcp_servers/logbook/logbook.py +++ b/src/seclab_taskflow_agent/mcp_servers/logbook/logbook.py @@ -2,17 +2,18 @@ # SPDX-License-Identifier: MIT import logging +#from mcp.server.fastmcp import FastMCP +from fastmcp import FastMCP # move to FastMCP 2.0 +import json +from pathlib import Path +from seclab_taskflow_agent.path_utils import mcp_data_dir, log_file_name + logging.basicConfig( level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s', - filename='logs/mcp_logbook.log', + filename=log_file_name('mcp_logbook.log'), filemode='a' ) -#from mcp.server.fastmcp import FastMCP -from fastmcp import FastMCP # move to FastMCP 2.0 -import json -from pathlib import Path -from seclab_taskflow_agent.path_utils import mcp_data_dir mcp = FastMCP("Logbook") diff --git a/src/seclab_taskflow_agent/mcp_servers/memcache/memcache.py b/src/seclab_taskflow_agent/mcp_servers/memcache/memcache.py index f8b706c..86b2029 100644 --- a/src/seclab_taskflow_agent/mcp_servers/memcache/memcache.py +++ b/src/seclab_taskflow_agent/mcp_servers/memcache/memcache.py @@ -2,12 +2,6 @@ # SPDX-License-Identifier: MIT import logging -logging.basicConfig( - level=logging.DEBUG, - format='%(asctime)s - %(levelname)s - %(message)s', - filename='logs/mcp_memcache.log', - filemode='a' -) #from mcp.server.fastmcp import FastMCP from fastmcp import FastMCP # move to FastMCP 2.0 import json @@ -16,7 +10,14 @@ from typing import Any from .memcache_backend.dictionary_file import MemcacheDictionaryFileBackend from .memcache_backend.sqlite import SqliteBackend -from seclab_taskflow_agent.path_utils import mcp_data_dir +from seclab_taskflow_agent.path_utils import mcp_data_dir, log_file_name + +logging.basicConfig( + level=logging.DEBUG, + format='%(asctime)s - %(levelname)s - %(message)s', + filename=log_file_name('mcp_memcache.log'), + filemode='a' +) mcp = FastMCP("Memcache") diff --git a/src/seclab_taskflow_agent/path_utils.py b/src/seclab_taskflow_agent/path_utils.py index 1175191..3d77b71 100644 --- a/src/seclab_taskflow_agent/path_utils.py +++ b/src/seclab_taskflow_agent/path_utils.py @@ -31,3 +31,11 @@ def mcp_data_dir(packagename: str, mcpname: str, env_override: str | None) -> Pa p = Path(d).joinpath(packagename).joinpath(mcpname) p.mkdir(parents=True, exist_ok=True) return p + +def log_dir(): + return platformdirs.user_log_dir(appname="seclab-taskflow-agent", + appauthor="GitHubSecurityLab", + ensure_exists=True) + +def log_file_name(filename: str): + return os.path.join(log_dir(), filename) diff --git a/src/seclab_taskflow_agent/render_utils.py b/src/seclab_taskflow_agent/render_utils.py index cbf0977..fdbe761 100644 --- a/src/seclab_taskflow_agent/render_utils.py +++ b/src/seclab_taskflow_agent/render_utils.py @@ -1,7 +1,16 @@ # SPDX-FileCopyrightText: 2025 GitHub # SPDX-License-Identifier: MIT +import logging import asyncio +from .path_utils import log_file_name + +logging.basicConfig( + level=logging.DEBUG, + format='%(asctime)s - %(levelname)s - %(message)s', + filename=log_file_name('render_stdout.log'), + filemode='a' +) async_output = {} async_output_lock = asyncio.Lock() @@ -31,6 +40,5 @@ async def render_model_output(data: str, data = "** 🤖✏️ Gathering output from async task ... please hold\n" if data: if log: - with open('logs/render_stdout.log', 'a') as f: - f.write(data) + logging.debug(data) print(data, end="", flush=True) From b1325f0b084699f1948646005cba4ddaa2533ab2 Mon Sep 17 00:00:00 2001 From: Kevin Backhouse Date: Tue, 2 Dec 2025 17:42:45 +0000 Subject: [PATCH 02/10] Update src/seclab_taskflow_agent/mcp_servers/codeql/client.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/seclab_taskflow_agent/mcp_servers/codeql/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/seclab_taskflow_agent/mcp_servers/codeql/client.py b/src/seclab_taskflow_agent/mcp_servers/codeql/client.py index 29a8637..605804e 100644 --- a/src/seclab_taskflow_agent/mcp_servers/codeql/client.py +++ b/src/seclab_taskflow_agent/mcp_servers/codeql/client.py @@ -12,7 +12,7 @@ import os import zipfile import yaml -from .path_utils import log_file_name +from seclab_taskflow_agent.path_utils import log_file_name # this is a local fork of https://github.com/riga/jsonrpyc modified for our purposes from . import jsonrpyc From 8c1ffbfa429d00f3a13ea7b6d6ea5fba2fc839aa Mon Sep 17 00:00:00 2001 From: Kevin Backhouse Date: Tue, 2 Dec 2025 17:42:58 +0000 Subject: [PATCH 03/10] Update src/seclab_taskflow_agent/mcp_servers/codeql/client.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/seclab_taskflow_agent/mcp_servers/codeql/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/seclab_taskflow_agent/mcp_servers/codeql/client.py b/src/seclab_taskflow_agent/mcp_servers/codeql/client.py index 605804e..9a1350c 100644 --- a/src/seclab_taskflow_agent/mcp_servers/codeql/client.py +++ b/src/seclab_taskflow_agent/mcp_servers/codeql/client.py @@ -46,7 +46,7 @@ def __init__(self, log_stderr=False): self.server_options = server_options.copy() if log_stderr: - self.stderr_log = log_file_name(codeql_stderr_log.log) + self.stderr_log = log_file_name('codeql_stderr_log.log') self.server_options.append("--log-to-stderr") else: self.stderr_log = os.devnull From a1ca2d332e2abc0d8d400b633730830c9583d780 Mon Sep 17 00:00:00 2001 From: Kevin Backhouse Date: Tue, 2 Dec 2025 17:43:33 +0000 Subject: [PATCH 04/10] Update src/seclab_taskflow_agent/mcp_servers/echo/echo.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/seclab_taskflow_agent/mcp_servers/echo/echo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/seclab_taskflow_agent/mcp_servers/echo/echo.py b/src/seclab_taskflow_agent/mcp_servers/echo/echo.py index c202f8f..eb063ed 100644 --- a/src/seclab_taskflow_agent/mcp_servers/echo/echo.py +++ b/src/seclab_taskflow_agent/mcp_servers/echo/echo.py @@ -4,7 +4,7 @@ import logging #from mcp.server.fastmcp import FastMCP from fastmcp import FastMCP # move to FastMCP 2.0 -from .path_utils import log_file_name +from seclab_taskflow_agent.path_utils import log_file_name logging.basicConfig( level=logging.DEBUG, From 385b41d61ec15f34f337efdff0c085582c757efa Mon Sep 17 00:00:00 2001 From: Kevin Backhouse Date: Tue, 2 Dec 2025 17:50:23 +0000 Subject: [PATCH 05/10] Update src/seclab_taskflow_agent/path_utils.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/seclab_taskflow_agent/path_utils.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/seclab_taskflow_agent/path_utils.py b/src/seclab_taskflow_agent/path_utils.py index 3d77b71..413467e 100644 --- a/src/seclab_taskflow_agent/path_utils.py +++ b/src/seclab_taskflow_agent/path_utils.py @@ -32,7 +32,13 @@ def mcp_data_dir(packagename: str, mcpname: str, env_override: str | None) -> Pa p.mkdir(parents=True, exist_ok=True) return p -def log_dir(): +def log_dir() -> str: + """ + Get the directory path for storing log files for the seclab-taskflow-agent. + + Returns: + str: The path to the log directory. + """ return platformdirs.user_log_dir(appname="seclab-taskflow-agent", appauthor="GitHubSecurityLab", ensure_exists=True) From b40ab4ba4bfc076a69cd3149abc01c3ee3dbf03b Mon Sep 17 00:00:00 2001 From: Kevin Backhouse Date: Tue, 2 Dec 2025 17:50:33 +0000 Subject: [PATCH 06/10] Update src/seclab_taskflow_agent/path_utils.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/seclab_taskflow_agent/path_utils.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/seclab_taskflow_agent/path_utils.py b/src/seclab_taskflow_agent/path_utils.py index 413467e..a498767 100644 --- a/src/seclab_taskflow_agent/path_utils.py +++ b/src/seclab_taskflow_agent/path_utils.py @@ -43,5 +43,14 @@ def log_dir() -> str: appauthor="GitHubSecurityLab", ensure_exists=True) -def log_file_name(filename: str): +def log_file_name(filename: str) -> str: + """ + Construct the full path to a log file in the user log directory. + + Parameters: + filename (str): The name of the log file. + + Returns: + str: The full path to the log file in the user log directory. + """ return os.path.join(log_dir(), filename) From 2d9c273eddab52d60d05d32c10d2ebcbd1ffc49a Mon Sep 17 00:00:00 2001 From: Kevin Backhouse Date: Wed, 3 Dec 2025 10:11:28 +0000 Subject: [PATCH 07/10] Allow default log directory location to be overridden with LOG_DIR environment variable. --- src/seclab_taskflow_agent/path_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/seclab_taskflow_agent/path_utils.py b/src/seclab_taskflow_agent/path_utils.py index a498767..031e9fa 100644 --- a/src/seclab_taskflow_agent/path_utils.py +++ b/src/seclab_taskflow_agent/path_utils.py @@ -39,6 +39,9 @@ def log_dir() -> str: Returns: str: The path to the log directory. """ + p = os.getenv("LOG_DIR") + if p: + return Path(p) return platformdirs.user_log_dir(appname="seclab-taskflow-agent", appauthor="GitHubSecurityLab", ensure_exists=True) From 81fb6798c17b386cff944f4d83bad6dae648ea83 Mon Sep 17 00:00:00 2001 From: Kevin Backhouse Date: Wed, 3 Dec 2025 11:47:47 +0000 Subject: [PATCH 08/10] Fix str/Path type confusion. --- src/seclab_taskflow_agent/path_utils.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/seclab_taskflow_agent/path_utils.py b/src/seclab_taskflow_agent/path_utils.py index 031e9fa..118e3ce 100644 --- a/src/seclab_taskflow_agent/path_utils.py +++ b/src/seclab_taskflow_agent/path_utils.py @@ -32,7 +32,7 @@ def mcp_data_dir(packagename: str, mcpname: str, env_override: str | None) -> Pa p.mkdir(parents=True, exist_ok=True) return p -def log_dir() -> str: +def log_dir() -> Path: """ Get the directory path for storing log files for the seclab-taskflow-agent. @@ -46,6 +46,18 @@ def log_dir() -> str: appauthor="GitHubSecurityLab", ensure_exists=True) +def log_file(filename: str) -> Path: + """ + Construct the full path to a log file in the user log directory. + + Parameters: + filename (str): The name of the log file. + + Returns: + Path: The full path to the log file in the user log directory. + """ + return log_dir().joinpath(filename) + def log_file_name(filename: str) -> str: """ Construct the full path to a log file in the user log directory. @@ -56,4 +68,4 @@ def log_file_name(filename: str) -> str: Returns: str: The full path to the log file in the user log directory. """ - return os.path.join(log_dir(), filename) + return str(log_file(filename)) From 9c80bfe2b131ffde346d2ba61db97068beee0cc7 Mon Sep 17 00:00:00 2001 From: Kevin Backhouse Date: Wed, 3 Dec 2025 11:50:21 +0000 Subject: [PATCH 09/10] Fix bug --- src/seclab_taskflow_agent/path_utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/seclab_taskflow_agent/path_utils.py b/src/seclab_taskflow_agent/path_utils.py index 118e3ce..0a431db 100644 --- a/src/seclab_taskflow_agent/path_utils.py +++ b/src/seclab_taskflow_agent/path_utils.py @@ -40,11 +40,11 @@ def log_dir() -> Path: str: The path to the log directory. """ p = os.getenv("LOG_DIR") - if p: - return Path(p) - return platformdirs.user_log_dir(appname="seclab-taskflow-agent", - appauthor="GitHubSecurityLab", - ensure_exists=True) + if not p: + p = platformdirs.user_log_dir(appname="seclab-taskflow-agent", + appauthor="GitHubSecurityLab", + ensure_exists=True) + return Path(p) def log_file(filename: str) -> Path: """ From 2f65245ad9fa48d233f323bd201425be89d92cba Mon Sep 17 00:00:00 2001 From: Kevin Backhouse Date: Wed, 3 Dec 2025 11:56:26 +0000 Subject: [PATCH 10/10] Update src/seclab_taskflow_agent/path_utils.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/seclab_taskflow_agent/path_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/seclab_taskflow_agent/path_utils.py b/src/seclab_taskflow_agent/path_utils.py index 0a431db..8d23347 100644 --- a/src/seclab_taskflow_agent/path_utils.py +++ b/src/seclab_taskflow_agent/path_utils.py @@ -37,7 +37,7 @@ def log_dir() -> Path: Get the directory path for storing log files for the seclab-taskflow-agent. Returns: - str: The path to the log directory. + Path: The path to the log directory. """ p = os.getenv("LOG_DIR") if not p: