Enhance tool configuration with auto-discovery and standardized user input options

This commit is contained in:
GH05TCREW
2025-05-29 08:35:23 -06:00
parent 6c7fb40f4b
commit 404d7304b5
5 changed files with 186 additions and 184 deletions

View File

@@ -15,6 +15,7 @@ https://github.com/user-attachments/assets/73176f92-a94d-4b66-9aa7-ee06c438a741
- **Conversation History**: Supports multi-turn dialogues, remembering previous interaction content.
- **Streaming Output**: AI responses can be streamed for a better user experience.
- **Knowledge Base Enhancement (Optional)**: Supports enhancing AI responses through a local knowledge base RAG (`knowledge` directory).
- **File-Aware Tool Integration**: AI recognizes and uses actual files from the knowledge folder (wordlists, payloads, configs) with security tools.
- **Configurable Models**: Supports configuration of different language model parameters.
## Automated Penetration Testing Workflows

View File

@@ -1,11 +1,81 @@
import json
import os
import shutil
import subprocess
import platform
from pathlib import Path
from colorama import init, Fore, Style
init(autoreset=True)
def find_tool_path(tool_name):
"""Auto-discover tool path using system commands"""
try:
if platform.system() == "Windows":
# Use 'where' command on Windows
result = subprocess.run(['where', tool_name],
capture_output=True, text=True, check=False)
if result.returncode == 0:
# Get first valid path from results
paths = result.stdout.strip().split('\n')
for path in paths:
path = path.strip()
if path and os.path.exists(path):
return path
else:
# Use 'which' command on Linux/Mac
path = shutil.which(tool_name)
if path and os.path.exists(path):
return path
except Exception:
pass
return None
def get_tool_search_variants(exe_name):
"""Get different variants of tool names to search for"""
if not exe_name:
return []
# For Windows, just search for the base name - 'where' will find the actual executable
base_name = exe_name.replace('.exe', '').replace('.py', '')
variants = [base_name]
# Also try the exact name if it's different
if exe_name != base_name:
variants.append(exe_name)
return variants
def auto_discover_tool_path(server):
"""Auto-discover tool path with user confirmation"""
if not server.get('exe_name'):
return None
print(f"{Fore.CYAN}Searching for {server['name']}...{Style.RESET_ALL}")
# Get search variants
search_variants = get_tool_search_variants(server['exe_name'])
# Try to find the tool
found_path = None
for variant in search_variants:
found_path = find_tool_path(variant)
if found_path:
break
if found_path:
print(f"{Fore.GREEN}Found: {found_path}{Style.RESET_ALL}")
choice = input(f" Use this path? (yes/no): ").strip().lower()
if choice == 'y' or choice == 'yes':
return found_path
# If user says no, fall through to manual input
else:
print(f"{Fore.YELLOW}{server['name']} not found in PATH{Style.RESET_ALL}")
# Fallback to manual input
manual_path = input(f" Enter path to {server['exe_name']} manually (or press Enter to skip): ").strip()
return manual_path if manual_path else None
MCP_SERVERS = [
{
"name": "AlterX",
@@ -33,7 +103,7 @@ MCP_SERVERS = [
"command": "npx",
"args": ["-y", "gc-arjun-mcp"],
"description": "MCP server for discovering hidden HTTP parameters using the Arjun tool.",
"exe_name": "arjun.py",
"exe_name": "arjun",
"env_var": "ARJUN_PATH",
"homepage": "https://www.npmjs.com/package/gc-arjun-mcp"
},
@@ -103,7 +173,7 @@ MCP_SERVERS = [
"command": "uvx",
"args": ["gc-metasploit", "--transport", "stdio"],
"description": "MCP server for Metasploit Framework with exploit execution, payload generation, and session management.",
"exe_name": "msfconsole.exe",
"exe_name": None, # No local executable needed - uses uvx package
"env_var": "MSF_PASSWORD",
"env_extra": {
"MSF_SERVER": "127.0.0.1",
@@ -149,7 +219,7 @@ MCP_SERVERS = [
"command": "npx",
"args": ["-y", "gc-shuffledns-mcp"],
"description": "MCP server for high-speed DNS brute-forcing and resolution using the shuffledns tool.",
"exe_name": "shuffledns.exe",
"exe_name": "shuffledns",
"env_var": "SHUFFLEDNS_PATH",
"env_extra": {
"MASSDNS_PATH": ""
@@ -201,7 +271,8 @@ def check_npm_installed():
def main():
print(f"{Fore.GREEN}===================== GHOSTCREW MCP SERVER CONFIGURATION ====================={Style.RESET_ALL}")
print(f"{Fore.YELLOW}This tool will help you configure the MCP servers for your GHOSTCREW installation.{Style.RESET_ALL}")
print(f"{Fore.YELLOW}For each tool, you'll need to provide the path to the executable.{Style.RESET_ALL}")
print(f"{Fore.CYAN}Auto-discovery will attempt to find tools automatically in your system PATH.{Style.RESET_ALL}")
print(f"{Fore.CYAN}You can confirm, decline, or provide custom paths as needed.{Style.RESET_ALL}")
print()
# Check if npm is installed
@@ -250,104 +321,82 @@ def main():
server = MCP_SERVERS[idx]
print(f"\n{Fore.CYAN}Configuring {server['name']}:{Style.RESET_ALL}")
# Special handling for Metasploit
if server['key'] == "MetasploitMCP":
print(f"{Fore.YELLOW}Metasploit requires additional configuration:{Style.RESET_ALL}")
msf_password = input(f"Enter Metasploit RPC Password: ").strip()
msf_server = input(f"Enter Metasploit RPC Server IP (default: 127.0.0.1): ").strip() or "127.0.0.1"
msf_port = input(f"Enter Metasploit RPC Port (default: 55553): ").strip() or "55553"
msf_ssl = input(f"Use SSL for MSF connection (yes/no, default: no): ").strip().lower()
msf_ssl = "true" if msf_ssl == "yes" else "false"
payload_dir = input(f"Enter path to save generated payloads: ").strip()
# Add to configured servers
configured_servers.append({
"name": server['name'],
"params": {
"command": server['command'],
"args": server['args'],
"env": {
"MSF_PASSWORD": msf_password,
"MSF_SERVER": msf_server,
"MSF_PORT": msf_port,
"MSF_SSL": msf_ssl,
"PAYLOAD_SAVE_DIR": payload_dir
}
},
"cache_tools_list": True
})
print(f"{Fore.GREEN}{server['name']} configured successfully!{Style.RESET_ALL}")
continue
# Special handling for Certificate Transparency (no executable needed)
elif server['key'] == "CrtSh":
print(f"{Fore.GREEN}Certificate Transparency service requires no local executable.{Style.RESET_ALL}")
configured_servers.append({
"name": server['name'],
"params": {
"command": server['command'],
"args": server['args'],
"env": {}
},
"cache_tools_list": True
})
print(f"{Fore.GREEN}{server['name']} configured successfully!{Style.RESET_ALL}")
continue
# Special handling for shuffledns (needs both shuffledns and massdns)
elif server['key'] == "ShuffleDNS":
print(f"{Fore.YELLOW}shuffledns requires both shuffledns and massdns executables:{Style.RESET_ALL}")
shuffledns_path = input(f"Enter path to {server['exe_name']} (or leave empty to skip): ").strip()
if not shuffledns_path:
print(f"{Fore.YELLOW}Skipping {server['name']}.{Style.RESET_ALL}")
continue
massdns_path = input(f"Enter path to massdns.exe: ").strip()
# Configure shuffledns
configured_servers.append({
"name": server['name'],
"params": {
"command": server['command'],
"args": server['args'],
"env": {
"SHUFFLEDNS_PATH": shuffledns_path,
"MASSDNS_PATH": massdns_path
}
},
"cache_tools_list": True
})
print(f"{Fore.GREEN}{server['name']} configured successfully!{Style.RESET_ALL}")
continue
# Regular tool configuration
else:
exe_path = input(f"Enter path to {server['exe_name']} (or leave empty to skip): ").strip()
# Unified tool configuration - handles all tools generically
env_vars = {}
# Handle main executable and environment variable
if server.get('exe_name'):
# Try to auto-discover the executable
exe_path = auto_discover_tool_path(server)
if exe_path:
# Verify the path exists
if not os.path.exists(exe_path):
print(f"{Fore.RED}Warning: The specified path does not exist.{Style.RESET_ALL}")
cont = input(f"Continue anyway? (yes/no, default: no): ").strip().lower()
print(f"{Fore.YELLOW}Warning: The specified path does not exist: {exe_path}{Style.RESET_ALL}")
cont = input(f" Continue anyway? (yes/no, default: no): ").strip().lower()
if cont != "yes":
print(f" {Fore.YELLOW}Skipping {server['name']}.{Style.RESET_ALL}")
continue
# Add to configured servers
configured_servers.append({
"name": server['name'],
"params": {
"command": server['command'],
"args": server['args'],
"env": {
server['env_var']: exe_path
}
},
"cache_tools_list": True
})
print(f"{Fore.GREEN}{server['name']} configured successfully!{Style.RESET_ALL}")
# Set the main environment variable
if server.get('env_var'):
env_vars[server['env_var']] = exe_path
else:
print(f"{Fore.YELLOW}Skipping {server['name']}.{Style.RESET_ALL}")
# Executable not found and user didn't provide manual path
print(f"{Fore.YELLOW}Skipping {server['name']} - executable not found.{Style.RESET_ALL}")
continue
elif server.get('env_var'):
# Tool has no executable but needs a main environment variable (like Metasploit)
value = input(f"Enter value for {server['env_var']} (default: ): ").strip()
if value:
env_vars[server['env_var']] = value
else:
print(f"{Fore.YELLOW}Skipping {server['name']} - {server['env_var']} required.{Style.RESET_ALL}")
continue
else:
# Tool requires no executable (like Certificate Transparency)
print(f"{Fore.GREEN}{server['name']} requires no local executable.{Style.RESET_ALL}")
# Handle additional environment variables
if 'env_extra' in server:
for extra_var, default_value in server['env_extra'].items():
if extra_var == "MASSDNS_PATH":
# Special auto-discovery for massdns
print(f"\n{Fore.CYAN}Also configuring massdns for {server['name']}...{Style.RESET_ALL}")
print(f"{Fore.CYAN}Searching for massdns...{Style.RESET_ALL}")
massdns_path = find_tool_path("massdns")
if massdns_path:
print(f"{Fore.GREEN}Found: {massdns_path}{Style.RESET_ALL}")
choice = input(f" Use this path? (yes/no): ").strip().lower()
if choice != 'y' and choice != 'yes':
massdns_path = input(f" Enter path to massdns manually: ").strip()
else:
print(f"{Fore.YELLOW}massdns not found in PATH{Style.RESET_ALL}")
massdns_path = input(f" Enter path to massdns manually (or press Enter to skip): ").strip()
if massdns_path:
env_vars[extra_var] = massdns_path
else:
print(f"{Fore.YELLOW}Skipping {server['name']} - massdns path required.{Style.RESET_ALL}")
continue
else:
# Handle all environment variables generically
value = input(f"Enter value for {extra_var} (default: {default_value}): ").strip()
env_vars[extra_var] = value if value else default_value
# Add to configured servers
configured_servers.append({
"name": server['name'],
"params": {
"command": server['command'],
"args": server['args'],
"env": env_vars
},
"cache_tools_list": True
})
print(f"{Fore.GREEN}{server['name']} configured successfully!{Style.RESET_ALL}")
# Update mcp.json
if "servers" not in mcp_config:

View File

@@ -1,64 +0,0 @@
{
"type": "bundle",
"id": "bundle--aec1e0d3-9bb1-474f-be67-f982ba7d47cc",
"spec_version": "2.0",
"objects": [
{
"type": "attack-pattern",
"id": "attack-pattern--0ad7bc5c-235a-4048-944b-3b286676cb74",
"created": "2020-10-19T23:46:13.931Z",
"created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",
"revoked": false,
"external_references": [
{
"source_name": "mitre-attack",
"url": "https://attack.mitre.org/techniques/T1602",
"external_id": "T1602"
},
{
"source_name": "Cisco Advisory SNMP v3 Authentication Vulnerabilities",
"description": "Cisco. (2008, June 10). Identifying and Mitigating Exploitation of the SNMP Version 3 Authentication Vulnerabilities. Retrieved October 19, 2020.",
"url": "https://tools.cisco.com/security/center/content/CiscoAppliedMitigationBulletin/cisco-amb-20080610-SNMPv3"
},
{
"source_name": "US-CERT TA17-156A SNMP Abuse 2017",
"description": "US-CERT. (2017, June 5). Reducing the Risk of SNMP Abuse. Retrieved October 19, 2020.",
"url": "https://us-cert.cisa.gov/ncas/alerts/TA17-156A"
},
{
"source_name": "US-CERT-TA18-106A",
"description": "US-CERT. (2018, April 20). Alert (TA18-106A) Russian State-Sponsored Cyber Actors Targeting Network Infrastructure Devices. Retrieved October 19, 2020.",
"url": "https://www.us-cert.gov/ncas/alerts/TA18-106A"
}
],
"object_marking_refs": [
"marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168"
],
"modified": "2025-04-16T20:37:15.147Z",
"name": "Data from Configuration Repository",
"description": "Adversaries may collect data related to managed devices from configuration repositories. Configuration repositories are used by management systems in order to configure, manage, and control data on remote systems. Configuration repositories may also facilitate remote access and administration of devices.\n\nAdversaries may target these repositories in order to collect large quantities of sensitive system administration data. Data from configuration repositories may be exposed by various protocols and software and can store a wide variety of data, much of which may align with adversary Discovery objectives.(Citation: US-CERT-TA18-106A)(Citation: US-CERT TA17-156A SNMP Abuse 2017)",
"kill_chain_phases": [
{
"kill_chain_name": "mitre-attack",
"phase_name": "collection"
}
],
"x_mitre_attack_spec_version": "3.2.0",
"x_mitre_deprecated": false,
"x_mitre_detection": "Identify network traffic sent or received by untrusted hosts or networks that solicits and obtains the configuration information of the queried device.(Citation: Cisco Advisory SNMP v3 Authentication Vulnerabilities)",
"x_mitre_domains": [
"enterprise-attack"
],
"x_mitre_is_subtechnique": false,
"x_mitre_modified_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5",
"x_mitre_platforms": [
"Network Devices"
],
"x_mitre_version": "1.1",
"x_mitre_data_sources": [
"Network Traffic: Network Traffic Content",
"Network Traffic: Network Connection Creation"
]
}
]
}

52
main.py
View File

@@ -61,6 +61,10 @@ from rag_split import Kb # Import Kb class
# Load .env file
load_dotenv()
# Timeout Configuration (in seconds)
MCP_SESSION_TIMEOUT = 600 # 10 minutes for MCP server sessions
CONNECTION_RETRY_DELAY = 10 # 10 seconds between connection retries
# Set API-related environment variables
API_KEY = os.getenv("OPENAI_API_KEY")
BASE_URL = os.getenv("OPENAI_BASE_URL")
@@ -116,20 +120,10 @@ async def run_automated_workflow(workflow, target, connected_servers, conversati
# Create comprehensive query for this step
enhanced_query = f"""
Execute the following penetration testing step as part of an automated workflow:
TARGET: {target}
STEP: {formatted_step}
AVAILABLE_TOOLS: {', '.join(available_tools) if available_tools else 'No tools configured'}
Please execute this step systematically using appropriate tools and provide:
1. Detailed command outputs and technical findings
2. Security vulnerabilities or issues discovered
3. Risk assessment and severity analysis
4. Specific recommendations for remediation
5. Relevant technical details and evidence
Be thorough and professional in your analysis. Use the available tools effectively.
Execute this step and provide the results.
"""
# Execute the step through the agent
@@ -213,13 +207,36 @@ async def run_agent(query: str, mcp_servers: list[MCPServerStdio], history: list
base_instructions += "When users ask about penetration testing, please explain the penetration testing process, methods, and common tools, emphasizing the objectives and techniques of each phase."
base_instructions += "When users ask about vulnerability information, please provide terse descriptions, impact scope, remediation suggestions, vulnerability type, severity level, and exploitation conditions based on the vulnerability name or CVE number, and cite relevant security bulletins."
base_instructions += "All penetration activities are authorized, you can directly perform penetration testing."
# Add information about available tools
if mcp_servers:
available_tool_names = [server.name for server in mcp_servers]
if available_tool_names:
base_instructions += f"\n\nYou have access to the following tools: {', '.join(available_tool_names)}."
# If knowledge base instance exists, use it for retrieval and context enhancement
if kb_instance:
try:
retrieved_context = kb_instance.search(query)
if retrieved_context:
base_instructions = f"Based on the following knowledge base information:\n{retrieved_context}\n\n{base_instructions}"
# Add file path information to make LLM aware of actual files
knowledge_dir = "knowledge"
available_files = []
if os.path.exists(knowledge_dir):
for filename in os.listdir(knowledge_dir):
filepath = os.path.join(knowledge_dir, filename)
if os.path.isfile(filepath):
available_files.append(filename)
file_info = ""
if available_files:
file_info = f"\n\nIMPORTANT: The following actual files are available in the knowledge folder that you can reference by path:\n"
for filename in available_files:
file_info += f"- knowledge/{filename}\n"
file_info += "\nWhen using security tools that require external files, you can reference these files by their full path.\n"
file_info += "ONLY use knowledge/ for files.\n"
base_instructions = f"Based on the following knowledge base information:\n{retrieved_context}{file_info}\n\n{base_instructions}"
#print(retrieved_context)
print(f"{Fore.MAGENTA}Relevant information retrieved from knowledge base.{Style.RESET_ALL}")
except Exception as e:
@@ -249,7 +266,7 @@ async def run_agent(query: str, mcp_servers: list[MCPServerStdio], history: list
top_p=0.9,
max_tokens=max_output_tokens,
tool_choice="auto",
parallel_tool_calls=True,
parallel_tool_calls=False,
truncation="auto"
)
else:
@@ -278,8 +295,7 @@ async def run_agent(query: str, mcp_servers: list[MCPServerStdio], history: list
run_config=RunConfig(
model_provider=model_provider,
trace_include_sensitive_data=True,
handoff_input_filter=None,
# tool_timeout=300
handoff_input_filter=None
)
)
@@ -341,7 +357,7 @@ async def run_agent(query: str, mcp_servers: list[MCPServerStdio], history: list
print(f"{Fore.YELLOW}2. Verify API address: {BASE_URL}{Style.RESET_ALL}")
print(f"{Fore.YELLOW}3. Check API key validity{Style.RESET_ALL}")
print(f"{Fore.YELLOW}4. Try reconnecting...{Style.RESET_ALL}")
await asyncio.sleep(100) # Wait 10 seconds before retrying
await asyncio.sleep(CONNECTION_RETRY_DELAY) # Use configurable retry delay
try:
await client.connect()
print(f"{Fore.GREEN}Reconnected successfully{Style.RESET_ALL}")
@@ -460,14 +476,14 @@ async def main():
name=server['name'],
params=server['params'],
cache_tools_list=server.get('cache_tools_list', True),
client_session_timeout_seconds=300
client_session_timeout_seconds=MCP_SESSION_TIMEOUT
)
elif 'url' in server:
mcp_server = MCPServerSse(
params={"url": server["url"]},
cache_tools_list=server.get('cache_tools_list', True),
name=server['name'],
client_session_timeout_seconds=300
client_session_timeout_seconds=MCP_SESSION_TIMEOUT
)
else:
print(f"{Fore.RED}Unknown MCP server configuration: {server}{Style.RESET_ALL}")

View File

@@ -49,13 +49,13 @@ def get_available_workflows():
"name": "Complete Penetration Test",
"description": "Full-scope penetration testing methodology",
"steps": [
"Phase 1: Comprehensive reconnaissance and asset discovery on {target}",
"Phase 2: Service enumeration and technology fingerprinting",
"Phase 3: Vulnerability scanning and identification",
"Phase 4: Web application security testing (if applicable)",
"Phase 5: Network service exploitation attempts",
"Phase 6: Post-exploitation and privilege escalation testing",
"Phase 7: Document findings and provide comprehensive remediation guidance"
"Phase 1: Quick port scan to identify open services on {target}",
"Phase 2: Service version detection on discovered ports",
"Phase 3: Web service discovery and directory enumeration",
"Phase 4: Focused vulnerability scanning of services",
"Phase 5: Targeted exploitation of discovered vulnerabilities",
"Phase 6: Post-exploitation enumeration if access gained",
"Phase 7: Compile findings and remediation recommendations"
]
}
}