diff --git a/.env.example b/.env.example index 902fc56..a590935 100644 --- a/.env.example +++ b/.env.example @@ -19,9 +19,35 @@ ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJhbm9uIiwKIC SERVICE_ROLE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJzZXJ2aWNlX3JvbGUiLAogICAgImlzcyI6ICJzdXBhYmFzZS1kZW1vIiwKICAgICJpYXQiOiAxNjQxNzY5MjAwLAogICAgImV4cCI6IDE3OTk1MzU2MDAKfQ.DaYlNEoUrrEn2Ig7tqibS-PHK5vgusbcbo7X36XVt4Q DASHBOARD_USERNAME=supabase DASHBOARD_PASSWORD=this_password_is_insecure_and_should_be_updated -SECRET_KEY_BASE=UpNVntn3cDxHJpq99YMc1T1AQgQpc8kfYTuRgBiYa15BLrx8etQoXz3gZv1/u2oq -VAULT_ENC_KEY=your-encryption-key-32-chars-min +POOLER_TENANT_ID=your-tenant-id +############ +# Caddy Config +############ + +# By default listen on https://localhost:[service port] and don't use an email for SSL +# To change this for production: +# Uncomment all of these environment variables for the services you want exposed +# Note that you might not want to expose Ollama or SearXNG since they aren't secured by default +# Replace the placeholder value with the host for each service (like n8n.yourdomain.com) +# Replace internal by your email (require to create a Let's Encrypt certificate) + +# N8N_HOSTNAME=n8n.yourdomain.com +# WEBUI_HOSTNAME=:openwebui.yourdomain.com +# FLOWISE_HOSTNAME=:flowise.yourdomain.com +# SUPABASE_HOSTNAME=:supabase.yourdomain.com +# OLLAMA_HOSTNAME=:ollama.yourdomain.com +# SEARXNG_HOSTNAME=searxng.yourdomain.com +# LETSENCRYPT_EMAIL=internal + +############ +# Optional SearXNG Config +# If you run a very small or a very large instance, you might want to change the amount of used uwsgi workers and threads per worker +# More workers (= processes) means that more search requests can be handled at the same time, but it also causes more resource usage +############ + +# SEARXNG_UWSGI_WORKERS=4 +# SEARXNG_UWSGI_THREADS=4 ############ # Database - You can change these to any PostgreSQL database that has logical replication enabled. @@ -34,12 +60,13 @@ POSTGRES_PORT=5432 ############ -# Supavisor -- Database pooler +# Supavisor -- Database pooler and others that can be left as default values ############ POOLER_PROXY_PORT_TRANSACTION=6543 POOLER_DEFAULT_POOL_SIZE=20 POOLER_MAX_CLIENT_CONN=100 -POOLER_TENANT_ID=your-tenant-id +SECRET_KEY_BASE=UpNVntn3cDxHJpq99YMc1T1AQgQpc8kfYTuRgBiYa15BLrx8etQoXz3gZv1/u2oq +VAULT_ENC_KEY=your-encryption-key-32-chars-min ############ diff --git a/.gitignore b/.gitignore index a3f8429..903b975 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,6 @@ .env.test volumes/ shared/ -supabase/ \ No newline at end of file +supabase/ +searxng/uwsgi.ini +searxng/settings.yml \ No newline at end of file diff --git a/Caddyfile b/Caddyfile new file mode 100644 index 0000000..387c816 --- /dev/null +++ b/Caddyfile @@ -0,0 +1,90 @@ +{ + # Global options - works for both environments + email {$LETSENCRYPT_EMAIL} +} + +# N8N +{$N8N_HOSTNAME} { + # For domains, Caddy will automatically use Let's Encrypt + # For localhost/port addresses, HTTPS won't be enabled + reverse_proxy localhost:5678 +} + +# Open WebUI +{$WEBUI_HOSTNAME} { + reverse_proxy localhost:3000 +} + +# Flowise +{$FLOWISE_HOSTNAME} { + reverse_proxy localhost:3001 +} + +# Ollama API +{$OLLAMA_HOSTNAME} { + reverse_proxy localhost:11434 +} + +# Supabase +{$SUPABASE_HOSTNAME} { + reverse_proxy localhost:8000 +} + +# SearXNG +{$SEARXNG_HOSTNAME} { + encode zstd gzip + + @api { + path /config + path /healthz + path /stats/errors + path /stats/checker + } + @search { + path /search + } + @imageproxy { + path /image_proxy + } + @static { + path /static/* + } + + header { + # CSP (https://content-security-policy.com) + Content-Security-Policy "upgrade-insecure-requests; default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; form-action 'self' https://github.com/searxng/searxng/issues/new; font-src 'self'; frame-ancestors 'self'; base-uri 'self'; connect-src 'self' https://overpass-api.de; img-src * data:; frame-src https://www.youtube-nocookie.com https://player.vimeo.com https://www.dailymotion.com https://www.deezer.com https://www.mixcloud.com https://w.soundcloud.com https://embed.spotify.com;" + # Disable some browser features + Permissions-Policy "accelerometer=(),camera=(),geolocation=(),gyroscope=(),magnetometer=(),microphone=(),payment=(),usb=()" + # Set referrer policy + Referrer-Policy "no-referrer" + # Force clients to use HTTPS + Strict-Transport-Security "max-age=31536000" + # Prevent MIME type sniffing from the declared Content-Type + X-Content-Type-Options "nosniff" + # X-Robots-Tag (comment to allow site indexing) + X-Robots-Tag "noindex, noarchive, nofollow" + # Remove "Server" header + -Server + } + + header @api { + Access-Control-Allow-Methods "GET, OPTIONS" + Access-Control-Allow-Origin "*" + } + + route { + # Cache policy + header Cache-Control "max-age=0, no-store" + header @search Cache-Control "max-age=5, private" + header @imageproxy Cache-Control "max-age=604800, public" + header @static Cache-Control "max-age=31536000, public, immutable" + } + + # SearXNG (uWSGI) + reverse_proxy localhost:8080 { + header_up X-Forwarded-Port {http.request.port} + header_up X-Real-IP {http.request.remote.host} + # https://github.com/searx/searx-docker/issues/24 + header_up Connection "close" + } +} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index a043a5a..282cd4d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,6 +4,9 @@ volumes: qdrant_storage: open-webui: flowise: + caddy-data: + caddy-config: + valkey-data: x-n8n: &service-n8n image: n8nio/n8n:latest @@ -100,6 +103,76 @@ services: volumes: - qdrant_storage:/qdrant/storage + caddy: + container_name: caddy + image: docker.io/library/caddy:2-alpine + network_mode: host + restart: unless-stopped + volumes: + - ./Caddyfile:/etc/caddy/Caddyfile:ro + - caddy-data:/data:rw + - caddy-config:/config:rw + environment: + - N8N_HOSTNAME=${N8N_HOSTNAME:-":8001"} + - WEBUI_HOSTNAME=${WEBUI_HOSTNAME:-":8002"} + - FLOWISE_HOSTNAME=${FLOWISE_HOSTNAME:-":8003"} + - OLLAMA_HOSTNAME=${OLLAMA_HOSTNAME:-":8004"} + - SUPABASE_HOSTNAME=${SUPABASE_HOSTNAME:-":8005"} + - SEARXNG_HOSTNAME=${SEARXNG_HOSTNAME:-":8006"} + - LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL:-internal} + cap_drop: + - ALL + cap_add: + - NET_BIND_SERVICE + logging: + driver: "json-file" + options: + max-size: "1m" + max-file: "1" + + redis: + container_name: redis + image: docker.io/valkey/valkey:8-alpine + command: valkey-server --save 30 1 --loglevel warning + restart: unless-stopped + volumes: + - valkey-data:/data + cap_drop: + - ALL + cap_add: + - SETGID + - SETUID + - DAC_OVERRIDE + logging: + driver: "json-file" + options: + max-size: "1m" + max-file: "1" + + searxng: + container_name: searxng + image: docker.io/searxng/searxng:latest + restart: unless-stopped + ports: + - 8080:8080 + volumes: + - ./searxng:/etc/searxng:rw + environment: + - SEARXNG_BASE_URL=https://${SEARXNG_HOSTNAME:-localhost}/ + - UWSGI_WORKERS=${SEARXNG_UWSGI_WORKERS:-4} + - UWSGI_THREADS=${SEARXNG_UWSGI_THREADS:-4} + cap_drop: + - ALL + cap_add: + - CHOWN + - SETGID + - SETUID + logging: + driver: "json-file" + options: + max-size: "1m" + max-file: "1" + ollama-cpu: profiles: ["cpu"] <<: *service-ollama diff --git a/searxng/settings-base.yml b/searxng/settings-base.yml new file mode 100644 index 0000000..3207869 --- /dev/null +++ b/searxng/settings-base.yml @@ -0,0 +1,11 @@ +# see https://docs.searxng.org/admin/settings/settings.html#settings-use-default-settings +use_default_settings: true +server: + # base_url is defined in the SEARXNG_BASE_URL environment variable, see .env and docker-compose.yml + secret_key: "ultrasecretkey" # change this! + limiter: true # can be disabled for a private instance + image_proxy: true +ui: + static_use_hash: true +redis: + url: redis://redis:6379/0 \ No newline at end of file diff --git a/start_services.py b/start_services.py index 82c7637..bac2c1a 100644 --- a/start_services.py +++ b/start_services.py @@ -12,6 +12,8 @@ import subprocess import shutil import time import argparse +import platform +import sys def run_command(cmd, cwd=None): """Run a shell command and print it.""" @@ -71,6 +73,146 @@ def start_local_ai(profile=None): cmd.extend(["-f", "docker-compose.yml", "up", "-d"]) run_command(cmd) +def generate_searxng_secret_key(): + """Generate a secret key for SearXNG based on the current platform.""" + print("Checking SearXNG settings...") + + # Define paths for SearXNG settings files + settings_path = os.path.join("searxng", "settings.yml") + settings_base_path = os.path.join("searxng", "settings-base.yml") + + # Check if settings-base.yml exists + if not os.path.exists(settings_base_path): + print(f"Warning: SearXNG base settings file not found at {settings_base_path}") + return + + # Check if settings.yml exists, if not create it from settings-base.yml + if not os.path.exists(settings_path): + print(f"SearXNG settings.yml not found. Creating from {settings_base_path}...") + try: + shutil.copyfile(settings_base_path, settings_path) + print(f"Created {settings_path} from {settings_base_path}") + except Exception as e: + print(f"Error creating settings.yml: {e}") + return + else: + print(f"SearXNG settings.yml already exists at {settings_path}") + + print("Generating SearXNG secret key...") + + # Detect the platform and run the appropriate command + system = platform.system() + + try: + if system == "Windows": + print("Detected Windows platform, using PowerShell to generate secret key...") + # PowerShell command to generate a random key and replace in the settings file + ps_command = [ + "powershell", "-Command", + "$randomBytes = New-Object byte[] 32; " + + "(New-Object Security.Cryptography.RNGCryptoServiceProvider).GetBytes($randomBytes); " + + "$secretKey = -join ($randomBytes | ForEach-Object { \"{0:x2}\" -f $_ }); " + + "(Get-Content searxng/settings.yml) -replace 'ultrasecretkey', $secretKey | Set-Content searxng/settings.yml" + ] + subprocess.run(ps_command, check=True) + + elif system == "Darwin": # macOS + print("Detected macOS platform, using sed command with empty string parameter...") + # macOS sed command requires an empty string for the -i parameter + openssl_cmd = ["openssl", "rand", "-hex", "32"] + random_key = subprocess.check_output(openssl_cmd).decode('utf-8').strip() + sed_cmd = ["sed", "-i", "", f"s|ultrasecretkey|{random_key}|g", settings_path] + subprocess.run(sed_cmd, check=True) + + else: # Linux and other Unix-like systems + print("Detected Linux/Unix platform, using standard sed command...") + # Standard sed command for Linux + openssl_cmd = ["openssl", "rand", "-hex", "32"] + random_key = subprocess.check_output(openssl_cmd).decode('utf-8').strip() + sed_cmd = ["sed", "-i", f"s|ultrasecretkey|{random_key}|g", settings_path] + subprocess.run(sed_cmd, check=True) + + print("SearXNG secret key generated successfully.") + + except Exception as e: + print(f"Error generating SearXNG secret key: {e}") + print("You may need to manually generate the secret key using the commands:") + print(" - Linux: sed -i \"s|ultrasecretkey|$(openssl rand -hex 32)|g\" searxng/settings.yml") + print(" - macOS: sed -i '' \"s|ultrasecretkey|$(openssl rand -hex 32)|g\" searxng/settings.yml") + print(" - Windows (PowerShell):") + print(" $randomBytes = New-Object byte[] 32") + print(" (New-Object Security.Cryptography.RNGCryptoServiceProvider).GetBytes($randomBytes)") + print(" $secretKey = -join ($randomBytes | ForEach-Object { \"{0:x2}\" -f $_ })") + print(" (Get-Content searxng/settings.yml) -replace 'ultrasecretkey', $secretKey | Set-Content searxng/settings.yml") + +def check_and_fix_docker_compose_for_searxng(): + """Check and modify docker-compose.yml for SearXNG first run.""" + docker_compose_path = "docker-compose.yml" + if not os.path.exists(docker_compose_path): + print(f"Warning: Docker Compose file not found at {docker_compose_path}") + return + + try: + # Read the docker-compose.yml file + with open(docker_compose_path, 'r') as file: + content = file.read() + + # Default to first run + is_first_run = True + + # Check if Docker is running and if the SearXNG container exists + try: + # Check if the SearXNG container is running + container_check = subprocess.run( + ["docker", "ps", "--filter", "name=searxng", "--format", "{{.Names}}"], + capture_output=True, text=True, check=True + ) + searxng_containers = container_check.stdout.strip().split('\n') + + # If SearXNG container is running, check inside for uwsgi.ini + if any(container for container in searxng_containers if container): + container_name = next(container for container in searxng_containers if container) + print(f"Found running SearXNG container: {container_name}") + + # Check if uwsgi.ini exists inside the container + container_check = subprocess.run( + ["docker", "exec", container_name, "sh", "-c", "[ -f /etc/searxng/uwsgi.ini ] && echo 'found' || echo 'not_found'"], + capture_output=True, text=True, check=True + ) + + if "found" in container_check.stdout: + print("Found uwsgi.ini inside the SearXNG container - not first run") + is_first_run = False + else: + print("uwsgi.ini not found inside the SearXNG container - first run") + is_first_run = True + else: + print("No running SearXNG container found - assuming first run") + except Exception as e: + print(f"Error checking Docker container: {e} - assuming first run") + + if is_first_run and "cap_drop: - ALL" in content: + print("First run detected for SearXNG. Temporarily removing 'cap_drop: - ALL' directive...") + # Temporarily comment out the cap_drop line + modified_content = content.replace("cap_drop: - ALL", "# cap_drop: - ALL # Temporarily commented out for first run") + + # Write the modified content back + with open(docker_compose_path, 'w') as file: + file.write(modified_content) + + print("Note: After the first run completes successfully, you should re-add 'cap_drop: - ALL' to docker-compose.yml for security reasons.") + elif not is_first_run and "# cap_drop: - ALL # Temporarily commented out for first run" in content: + print("SearXNG has been initialized. Re-enabling 'cap_drop: - ALL' directive for security...") + # Uncomment the cap_drop line + modified_content = content.replace("# cap_drop: - ALL # Temporarily commented out for first run", "cap_drop: - ALL") + + # Write the modified content back + with open(docker_compose_path, 'w') as file: + file.write(modified_content) + + except Exception as e: + print(f"Error checking/modifying docker-compose.yml for SearXNG: {e}") + def main(): parser = argparse.ArgumentParser(description='Start the local AI and Supabase services.') parser.add_argument('--profile', choices=['cpu', 'gpu-nvidia', 'gpu-amd', 'none'], default='cpu', @@ -79,6 +221,11 @@ def main(): clone_supabase_repo() prepare_supabase_env() + + # Generate SearXNG secret key and check docker-compose.yml + generate_searxng_secret_key() + check_and_fix_docker_compose_for_searxng() + stop_existing_containers() # Start Supabase first