diff --git a/docker-compose.yml b/docker-compose.yml index 9472bee..a043a5a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,6 +36,10 @@ x-init-ollama: &init-ollama command: - "-c" - "sleep 3; OLLAMA_HOST=ollama:11434 ollama pull qwen2.5:7b-instruct-q4_K_M; OLLAMA_HOST=ollama:11434 ollama pull nomic-embed-text" + # For a larger context length verison of the model, run these commands: + # echo "FROM qwen2.5:7b-instruct-q4_K_M\n\nPARAMETER num_ctx 8096" > Modelfile + # ollama create qwen2.5:7b-8k -f ./Modelfile + # Change the name of the LLM and num_ctx as you see fit. services: flowise: diff --git a/n8n/backup/workflows/Local_RAG_AI_Agent_n8n_Workflow.json b/n8n/backup/workflows/V1_Local_RAG_AI_Agent.json similarity index 99% rename from n8n/backup/workflows/Local_RAG_AI_Agent_n8n_Workflow.json rename to n8n/backup/workflows/V1_Local_RAG_AI_Agent.json index 3ebc4e4..cdca711 100644 --- a/n8n/backup/workflows/Local_RAG_AI_Agent_n8n_Workflow.json +++ b/n8n/backup/workflows/V1_Local_RAG_AI_Agent.json @@ -1,5 +1,5 @@ { - "name": "Local RAG AI Agent", + "name": "V1 ocal RAG AI Agent", "nodes": [ { "parameters": {}, diff --git a/n8n/backup/workflows/Local_Supabase_RAG_AI_Agent_Workflow.json b/n8n/backup/workflows/V2_Local_Supabase_RAG_AI_Agent.json similarity index 86% rename from n8n/backup/workflows/Local_Supabase_RAG_AI_Agent_Workflow.json rename to n8n/backup/workflows/V2_Local_Supabase_RAG_AI_Agent.json index 279876c..40287c8 100644 --- a/n8n/backup/workflows/Local_Supabase_RAG_AI_Agent_Workflow.json +++ b/n8n/backup/workflows/V2_Local_Supabase_RAG_AI_Agent.json @@ -1,15 +1,15 @@ { - "name": "Local Supabase RAG AI Agent", + "name": "V2 Supabase RAG AI Agent", "nodes": [ { "parameters": {}, - "id": "99b30fd7-b36c-44ba-9daa-408585aaaee9", + "id": "3e70b57d-49fb-4cb0-8f9f-29d39adf6a65", "name": "Postgres Chat Memory", "type": "@n8n/n8n-nodes-langchain.memoryPostgresChat", "typeVersion": 1.1, "position": [ - 1040, - 560 + 480, + 340 ], "credentials": { "postgres": { @@ -23,13 +23,13 @@ "model": "qwen2.5:7b-instruct-q4_K_M", "options": {} }, - "id": "c7632a7c-2661-492e-bd6f-aab994818998", + "id": "8d61de27-45d8-4d10-97cc-3c36d224f865", "name": "Ollama Chat Model", "type": "@n8n/n8n-nodes-langchain.lmChatOllama", "typeVersion": 1, "position": [ - 920, - 560 + 360, + 340 ], "credentials": { "ollamaApi": { @@ -43,13 +43,13 @@ "model": "qwen2.5:7b-instruct-q4_K_M", "options": {} }, - "id": "73d773a4-5c72-4af3-a52d-144f0e417823", + "id": "cbd5b56f-7afc-4e83-a221-6be4d348374e", "name": "Ollama Model", "type": "@n8n/n8n-nodes-langchain.lmOllama", "typeVersion": 1, "position": [ - 1960, - 500 + 1400, + 280 ], "credentials": { "ollamaApi": { @@ -63,13 +63,13 @@ "name": "documents", "topK": 3 }, - "id": "3f882fa7-c8ed-4531-b236-a34c16c55838", + "id": "c15bd0a8-286a-4076-8ce8-8c0e54f73e2a", "name": "Vector Store Tool", "type": "@n8n/n8n-nodes-langchain.toolVectorStore", "typeVersion": 1, "position": [ - 1740, - 320 + 1180, + 100 ] }, { @@ -78,13 +78,13 @@ "destinationKey": "=data", "options": {} }, - "id": "7efee822-68ad-4fe2-a616-ba19fd127684", + "id": "ab9fb44a-85c6-486b-b5de-0f9d768d91b2", "name": "Extract Document Text", "type": "n8n-nodes-base.extractFromFile", "typeVersion": 1, "position": [ - 1480, - 860 + 920, + 640 ], "alwaysOutputData": true }, @@ -101,13 +101,13 @@ } } }, - "id": "da4c8b29-4944-43c4-9df3-e380366c594a", + "id": "b5abf0a5-f62a-49a6-bf81-bf002ba4bb90", "name": "Default Data Loader", "type": "@n8n/n8n-nodes-langchain.documentDefaultDataLoader", "typeVersion": 1, "position": [ - 1860, - 1100 + 1300, + 880 ] }, { @@ -115,26 +115,26 @@ "chunkSize": 100, "options": {} }, - "id": "d11c39b9-3fa7-4d5d-838f-da0d258c67c5", + "id": "1e2791c1-86e7-4fe1-a76d-732f87e6f41a", "name": "Recursive Character Text Splitter", "type": "@n8n/n8n-nodes-langchain.textSplitterRecursiveCharacterTextSplitter", "typeVersion": 1, "position": [ - 1860, - 1320 + 1300, + 1100 ] }, { "parameters": { "model": "nomic-embed-text:latest" }, - "id": "8a04559c-dfe8-479f-8998-a2e9bc994a0a", + "id": "909713ea-5be3-4916-b12e-a8e848c949cb", "name": "Embeddings Ollama1", "type": "@n8n/n8n-nodes-langchain.embeddingsOllama", "typeVersion": 1, "position": [ - 1700, - 1100 + 1140, + 880 ], "credentials": { "ollamaApi": { @@ -149,13 +149,13 @@ "height": 527.3027193303974, "width": 969.0343804425795 }, - "id": "a18773ae-1eb3-46b8-91cf-4184c66cf14f", + "id": "de473f2f-b806-45bc-a0e0-2ef0d2aa9b55", "name": "Sticky Note2", "type": "n8n-nodes-base.stickyNote", "typeVersion": 1, "position": [ - 560, - 220 + 0, + 0 ] }, { @@ -165,13 +165,13 @@ "width": 583.4552380860637, "color": 4 }, - "id": "fa010a11-3dda-4bd5-b261-463a3a6b88d9", + "id": "b23ef570-f642-4503-87b9-3494ffdbf768", "name": "Sticky Note", "type": "n8n-nodes-base.stickyNote", "typeVersion": 1, "position": [ - 1540, - 220 + 980, + 0 ] }, { @@ -181,26 +181,26 @@ "width": 1568.9362829025763, "color": 5 }, - "id": "f29e6cc7-015e-47cb-a4fd-fecd6ffb0d24", + "id": "836cd765-dae8-460e-951d-66e19d0cce77", "name": "Sticky Note1", "type": "n8n-nodes-base.stickyNote", "typeVersion": 1, "position": [ - 560, - 760 + 0, + 540 ] }, { "parameters": { "options": {} }, - "id": "5da52326-dfbd-4350-919c-843461f58913", + "id": "f8b5039c-4b00-453a-b30e-31a59f5d36ad", "name": "When chat message received", "type": "@n8n/n8n-nodes-langchain.chatTrigger", "typeVersion": 1.1, "position": [ - 620, - 340 + 60, + 120 ], "webhookId": "4b3b1838-d6b3-447e-9d79-d0931eddb9f8" }, @@ -208,13 +208,13 @@ "parameters": { "options": {} }, - "id": "e537544a-37d5-4b00-b5ff-bc71f041f4bb", + "id": "2f64907b-42ef-4bd6-83ba-e97584271bc7", "name": "Respond to Webhook", "type": "n8n-nodes-base.respondToWebhook", "typeVersion": 1.1, "position": [ - 1340, - 340 + 780, + 120 ] }, { @@ -224,13 +224,13 @@ "responseMode": "responseNode", "options": {} }, - "id": "2b8cd01f-30a8-4aab-b0dd-56d2b658f059", + "id": "9a30041f-0c14-41d8-a811-229b890bb1b7", "name": "Webhook", "type": "n8n-nodes-base.webhook", "typeVersion": 2, "position": [ - 620, - 520 + 60, + 300 ], "webhookId": "4a839da9-b8a2-45f8-bcaf-c484f9a5912d" }, @@ -238,13 +238,13 @@ "parameters": { "options": {} }, - "id": "c9dfe906-178b-4375-8bda-f9290f35f222", + "id": "8311e22f-bddd-41f8-9d40-fde119126dc9", "name": "AI Agent", "type": "@n8n/n8n-nodes-langchain.agent", "typeVersion": 1.6, "position": [ - 1000, - 340 + 440, + 120 ] }, { @@ -267,13 +267,13 @@ }, "options": {} }, - "id": "8f974a15-aa2f-4525-8278-ad58ad296076", + "id": "4988a14f-a2ea-4c4b-8f66-0f1773e11ea0", "name": "Edit Fields", "type": "n8n-nodes-base.set", "typeVersion": 3.4, "position": [ - 820, - 340 + 260, + 120 ] }, { @@ -291,10 +291,10 @@ "type": "@n8n/n8n-nodes-langchain.vectorStoreSupabase", "typeVersion": 1, "position": [ - 1560, - 460 + 1000, + 240 ], - "id": "2263db0a-944b-4fe3-9c3e-f1546fb5da97", + "id": "9841944a-5a85-437d-a9ed-a3e7393d7a8d", "name": "Supabase Vector Store", "credentials": { "supabaseApi": { @@ -310,10 +310,10 @@ "type": "@n8n/n8n-nodes-langchain.embeddingsOllama", "typeVersion": 1, "position": [ - 1740, - 600 + 1180, + 380 ], - "id": "3f4477b6-9708-4bcd-a774-515b3362ef47", + "id": "24e6e703-33c2-426d-ab59-7c9cad0fded9", "name": "Embeddings Ollama2", "credentials": { "ollamaApi": { @@ -338,10 +338,10 @@ "type": "@n8n/n8n-nodes-langchain.vectorStoreSupabase", "typeVersion": 1, "position": [ - 1740, - 860 + 1180, + 640 ], - "id": "9a8d05de-2305-45cd-ac97-5b6658058471", + "id": "d72c6f38-7766-4a10-9897-362539a6bcc0", "name": "Supabase Vector Store1", "credentials": { "supabaseApi": { @@ -357,13 +357,13 @@ "filterType": "string", "filterString": "=metadata->>file_id=like.*{{ $json.path }}*" }, - "id": "60534fbb-e620-42f1-a4da-e948694a324f", + "id": "ab0e0395-9f88-438f-bc47-ea3913b869fe", "name": "Delete Old Doc Rows", "type": "n8n-nodes-base.supabase", "typeVersion": 1, "position": [ - 1020, - 1060 + 460, + 840 ], "alwaysOutputData": true, "credentials": { @@ -389,15 +389,15 @@ "type": "n8n-nodes-base.localFileTrigger", "typeVersion": 1, "position": [ - 620, - 1060 + 60, + 840 ], - "id": "d3ccc49c-28dd-41bc-ae86-752ab6d38a0b", + "id": "eba68fe1-738d-451b-a2a7-9ee2942dd727", "name": "Local File Trigger" }, { "parameters": { - "fileSelector": "/data/shared/test.txt", + "fileSelector": "={{ $('Local File Trigger').item.json.path }}", "options": { "dataPropertyName": "=data" } @@ -405,11 +405,12 @@ "type": "n8n-nodes-base.readWriteFile", "typeVersion": 1, "position": [ - 1200, - 860 + 640, + 640 ], - "id": "1838461b-8783-4dc1-b7a0-f7bd14499135", - "name": "Read/Write Files from Disk" + "id": "b9459d4d-836c-47c4-9651-ebf3129f8864", + "name": "Read/Write Files from Disk", + "executeOnce": true }, { "parameters": { @@ -439,10 +440,10 @@ "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ - 840, - 880 + 280, + 660 ], - "id": "d94e79a0-1bf2-4f44-b2cc-f8bad1bc30f5", + "id": "27123d6a-e27a-49a9-bd73-8a27235928ea", "name": "If" } ], @@ -658,11 +659,10 @@ "settings": { "executionOrder": "v1" }, - "versionId": "26d69fe7-6473-410d-a553-a799457c2157", + "versionId": "ea9ff68c-8fc0-40b0-aa5d-48217cda89f3", "meta": { - "templateCredsSetupCompleted": true, "instanceId": "73cb7a3e883df514bb47e8d1b34526d30e2abb8f56cd99f10d5948a1e11b25aa" }, - "id": "vTN9y2dLXqTiDfPT", + "id": "hrnPh6dXgIbGVzIk", "tags": [] } \ No newline at end of file diff --git a/n8n/backup/workflows/V3_Local_Agentic_RAG_AI_Agent.json b/n8n/backup/workflows/V3_Local_Agentic_RAG_AI_Agent.json new file mode 100644 index 0000000..0756f1a --- /dev/null +++ b/n8n/backup/workflows/V3_Local_Agentic_RAG_AI_Agent.json @@ -0,0 +1,1493 @@ +{ + "name": "V3 Local Agentic RAG AI Agent", + "nodes": [ + { + "parameters": { + "jsonMode": "expressionData", + "jsonData": "={{ $json.data || $json.text || $json.concatenated_data }}", + "options": { + "metadata": { + "metadataValues": [ + { + "name": "=file_id", + "value": "={{ $('Set File ID').first().json.file_id }}" + }, + { + "name": "file_title", + "value": "={{ $('Set File ID').first().json.file_title }}" + } + ] + } + } + }, + "id": "55f9647b-a2b2-4b19-97d2-ddc8864e440b", + "name": "Default Data Loader", + "type": "@n8n/n8n-nodes-langchain.documentDefaultDataLoader", + "typeVersion": 1, + "position": [ + 1000, + 1140 + ] + }, + { + "parameters": { + "content": "## Agent Tools for RAG", + "height": 528.85546469693, + "width": 583.4552380860637, + "color": 4 + }, + "id": "ceb2c823-1cce-4813-85e6-c3e638e68091", + "name": "Sticky Note", + "type": "n8n-nodes-base.stickyNote", + "typeVersion": 1, + "position": [ + 0, + 0 + ] + }, + { + "parameters": { + "content": "## Tool to Add a Google Drive File to Vector DB", + "height": 867, + "width": 3073, + "color": 5 + }, + "id": "7bbc9f6b-ac99-471f-8bce-72427a4f775e", + "name": "Sticky Note1", + "type": "n8n-nodes-base.stickyNote", + "typeVersion": 1, + "position": [ + -1740, + 540 + ] + }, + { + "parameters": { + "operation": "text", + "options": {} + }, + "id": "f80a6769-d48d-4bd9-a6fc-ca718a2ef4b9", + "name": "Extract Document Text", + "type": "n8n-nodes-base.extractFromFile", + "typeVersion": 1, + "position": [ + 200, + 1140 + ], + "alwaysOutputData": true + }, + { + "parameters": {}, + "id": "1f5d7294-b47e-41fe-99d7-7f79617c76cd", + "name": "Postgres Chat Memory", + "type": "@n8n/n8n-nodes-langchain.memoryPostgresChat", + "typeVersion": 1, + "position": [ + -600, + 360 + ], + "notesInFlow": false, + "credentials": { + "postgres": { + "id": "UaTmh0frrACTMPxG", + "name": "Postgres account" + } + } + }, + { + "parameters": { + "assignments": { + "assignments": [ + { + "id": "10646eae-ae46-4327-a4dc-9987c2d76173", + "name": "file_id", + "value": "={{ $json.path }}", + "type": "string" + }, + { + "id": "f4536df5-d0b1-4392-bf17-b8137fb31a44", + "name": "file_type", + "value": "={{ $json.path.split(/[\\\\/]/).pop().split('.').pop(); }}", + "type": "string" + }, + { + "id": "77d782de-169d-4a46-8a8e-a3831c04d90f", + "name": "file_title", + "value": "={{ $json.path.split(/[\\\\/]/).pop().split('.').slice(0, -1).join('.'); }}", + "type": "string" + } + ] + }, + "options": {} + }, + "id": "5f283705-2576-4756-88fb-120463c3b69b", + "name": "Set File ID", + "type": "n8n-nodes-base.set", + "typeVersion": 3.4, + "position": [ + -1320, + 820 + ] + }, + { + "parameters": { + "content": "## RAG AI Agent with Chat Interface", + "height": 464.8027193303974, + "width": 1035.6381264595484 + }, + "id": "dc23d387-98e7-421d-b277-60dfbe06c501", + "name": "Sticky Note2", + "type": "n8n-nodes-base.stickyNote", + "typeVersion": 1, + "position": [ + -1040, + 60 + ] + }, + { + "parameters": { + "options": {} + }, + "id": "f285aab4-53d1-46a9-9e22-2686f082a374", + "name": "Respond to Webhook", + "type": "n8n-nodes-base.respondToWebhook", + "typeVersion": 1.1, + "position": [ + -180, + 140 + ] + }, + { + "parameters": { + "assignments": { + "assignments": [ + { + "id": "9a9a245e-f1a1-4282-bb02-a81ffe629f0f", + "name": "chatInput", + "value": "={{ $json?.chatInput || $json.body.chatInput }}", + "type": "string" + }, + { + "id": "b80831d8-c653-4203-8706-adedfdb98f77", + "name": "sessionId", + "value": "={{ $json?.sessionId || $json.body.sessionId}}", + "type": "string" + } + ] + }, + "options": {} + }, + "id": "129c23bf-c9dc-4b87-820f-3f371f5a420d", + "name": "Edit Fields", + "type": "n8n-nodes-base.set", + "typeVersion": 3.4, + "position": [ + -740, + 140 + ] + }, + { + "parameters": { + "public": true, + "options": {} + }, + "id": "fffe770f-5fe3-4ded-8aed-5d628a5e622f", + "name": "When chat message received", + "type": "@n8n/n8n-nodes-langchain.chatTrigger", + "typeVersion": 1.1, + "position": [ + -1000, + 140 + ], + "webhookId": "e104e40e-6134-4825-a6f0-8a646d882662" + }, + { + "parameters": { + "httpMethod": "POST", + "path": "bf4dd093-bb02-472c-9454-7ab9af97bd1d", + "responseMode": "responseNode", + "options": {} + }, + "id": "5eb1415f-4d37-4e63-8914-f370ae12d275", + "name": "Webhook", + "type": "n8n-nodes-base.webhook", + "typeVersion": 2, + "position": [ + -1000, + 340 + ], + "webhookId": "bf4dd093-bb02-472c-9454-7ab9af97bd1d" + }, + { + "parameters": { + "operation": "pdf", + "options": {} + }, + "id": "d050553b-da5c-4250-8f25-0300f49099cc", + "name": "Extract PDF Text", + "type": "n8n-nodes-base.extractFromFile", + "typeVersion": 1, + "position": [ + 200, + 580 + ] + }, + { + "parameters": { + "aggregate": "aggregateAllItemData", + "options": {} + }, + "id": "8f5f530e-186a-4788-86d1-cb41d0400286", + "name": "Aggregate", + "type": "n8n-nodes-base.aggregate", + "typeVersion": 1, + "position": [ + 240, + 760 + ] + }, + { + "parameters": { + "fieldsToSummarize": { + "values": [ + { + "aggregation": "concatenate", + "field": "data" + } + ] + }, + "options": {} + }, + "id": "50cb0e47-5857-4a27-a1a4-12357ba50420", + "name": "Summarize", + "type": "n8n-nodes-base.summarize", + "typeVersion": 1, + "position": [ + 440, + 840 + ] + }, + { + "parameters": { + "promptType": "define", + "text": "={{ $json.chatInput }}", + "options": { + "systemMessage": "You are a personal assistant who helps answer questions from a corpus of documents. The documents are either text based (Txt, docs, extracted PDFs, etc.) or tabular data (CSVs or Excel documents).\n\nYou are given tools to perform RAG in the 'documents' table, look up the documents available in your knowledge base in the 'document_metadata' table, extract all the text from a given document, and query the tabular files with SQL in the 'document_rows' table.\n\nAlways start by performing RAG unless the question requires a SQL query for tabular data (fetching a sum, finding a max, something a RAG lookup would be unreliable for). If RAG doesn't help, then look at the documents that are available to you, find a few that you think would contain the answer, and then analyze those.\n\nAlways tell the user if you didn't find the answer. Don't make something up just to please them." + } + }, + "id": "8d6daa23-a646-4592-a449-02cf50e2c953", + "name": "RAG AI Agent", + "type": "@n8n/n8n-nodes-langchain.agent", + "typeVersion": 1.6, + "position": [ + -520, + 140 + ] + }, + { + "parameters": { + "rules": { + "values": [ + { + "conditions": { + "options": { + "caseSensitive": true, + "leftValue": "", + "typeValidation": "strict", + "version": 1 + }, + "conditions": [ + { + "leftValue": "={{ $('Set File ID').item.json.file_type }}", + "rightValue": "pdf", + "operator": { + "type": "string", + "operation": "equals" + } + } + ], + "combinator": "and" + } + }, + { + "conditions": { + "options": { + "caseSensitive": true, + "leftValue": "", + "typeValidation": "strict", + "version": 1 + }, + "conditions": [ + { + "id": "2ae7faa7-a936-4621-a680-60c512163034", + "leftValue": "={{ $('Set File ID').item.json.file_type }}", + "rightValue": "xlsx", + "operator": { + "type": "string", + "operation": "equals", + "name": "filter.operator.equals" + } + } + ], + "combinator": "and" + } + }, + { + "conditions": { + "options": { + "caseSensitive": true, + "leftValue": "", + "typeValidation": "strict", + "version": 1 + }, + "conditions": [ + { + "id": "fc193b06-363b-4699-a97d-e5a850138b0e", + "leftValue": "={{ $('Set File ID').item.json.file_type }}", + "rightValue": "=csv", + "operator": { + "type": "string", + "operation": "equals", + "name": "filter.operator.equals" + } + } + ], + "combinator": "and" + } + }, + { + "conditions": { + "options": { + "caseSensitive": true, + "leftValue": "", + "typeValidation": "strict", + "version": 1 + }, + "conditions": [ + { + "id": "b69f5605-0179-4b02-9a32-e34bb085f82d", + "leftValue": "={{ $('Set File ID').item.json.file_type }}", + "rightValue": "txt", + "operator": { + "type": "string", + "operation": "equals", + "name": "filter.operator.equals" + } + } + ], + "combinator": "and" + } + } + ] + }, + "options": { + "fallbackOutput": 3 + } + }, + "id": "a381554d-5628-44dc-8e8c-fae60d6860b0", + "name": "Switch", + "type": "n8n-nodes-base.switch", + "typeVersion": 3, + "position": [ + -460, + 800 + ] + }, + { + "parameters": { + "operation": "xlsx", + "options": {} + }, + "id": "713308e3-f35f-4657-b6fb-06882b178102", + "name": "Extract from Excel", + "type": "n8n-nodes-base.extractFromFile", + "typeVersion": 1, + "position": [ + 20, + 760 + ] + }, + { + "parameters": { + "assignments": { + "assignments": [ + { + "id": "f422e2e0-381c-46ea-8f38-3f58c501d8b9", + "name": "schema", + "value": "={{ $('Extract from Excel').isExecuted ? $('Extract from Excel').first().json.keys().toJsonString() : $('Extract from CSV').first().json.keys().toJsonString() }}", + "type": "string" + }, + { + "id": "bb07c71e-5b60-4795-864c-cc3845b6bc46", + "name": "data", + "value": "={{ $json.concatenated_data }}", + "type": "string" + } + ] + }, + "options": {} + }, + "type": "n8n-nodes-base.set", + "typeVersion": 3.4, + "position": [ + 880, + 700 + ], + "id": "068f0564-7f2b-40df-8a82-ddfff6ac3192", + "name": "Set Schema" + }, + { + "parameters": { + "options": {} + }, + "type": "n8n-nodes-base.extractFromFile", + "typeVersion": 1, + "position": [ + 20, + 940 + ], + "id": "9cc88ee0-a065-4bee-a6d8-135c433da188", + "name": "Extract from CSV" + }, + { + "parameters": { + "content": "## Run Each Node Once to Set Up Database Tables", + "height": 300, + "width": 680, + "color": 3 + }, + "type": "n8n-nodes-base.stickyNote", + "position": [ + -1740, + 220 + ], + "typeVersion": 1, + "id": "452a79b7-d5fe-4d5d-b32f-9df1d714423d", + "name": "Sticky Note3" + }, + { + "parameters": { + "operation": "executeQuery", + "query": "CREATE TABLE document_metadata (\n id TEXT PRIMARY KEY,\n title TEXT,\n created_at TIMESTAMP DEFAULT NOW(),\n schema TEXT\n);", + "options": {} + }, + "type": "n8n-nodes-base.postgres", + "typeVersion": 2.5, + "position": [ + -1620, + 320 + ], + "id": "8d137c72-d6b5-4f91-a114-8d9f1777411b", + "name": "Create Document Metadata Table", + "credentials": { + "postgres": { + "id": "UaTmh0frrACTMPxG", + "name": "Postgres account" + } + } + }, + { + "parameters": { + "operation": "executeQuery", + "query": "CREATE TABLE document_rows (\n id SERIAL PRIMARY KEY,\n dataset_id TEXT REFERENCES document_metadata(id),\n row_data JSONB -- Store the actual row data\n);", + "options": {} + }, + "type": "n8n-nodes-base.postgres", + "typeVersion": 2.5, + "position": [ + -1320, + 320 + ], + "id": "ddd63d7b-b7e7-4af5-9b2d-72898ca9b63a", + "name": "Create Document Rows Table (for Tabular Data)", + "credentials": { + "postgres": { + "id": "UaTmh0frrACTMPxG", + "name": "Postgres account" + } + } + }, + { + "parameters": { + "descriptionType": "manual", + "toolDescription": "Use this tool to fetch all available documents, including the table schema if the file is a CSV or Excel file.", + "operation": "select", + "schema": { + "__rl": true, + "mode": "list", + "value": "public" + }, + "table": { + "__rl": true, + "value": "document_metadata", + "mode": "list", + "cachedResultName": "document_metadata" + }, + "returnAll": true, + "options": {} + }, + "type": "n8n-nodes-base.postgresTool", + "typeVersion": 2.5, + "position": [ + -460, + 360 + ], + "id": "e5223ab2-36fe-4294-9acc-14acefa716e4", + "name": "List Documents", + "credentials": { + "postgres": { + "id": "UaTmh0frrACTMPxG", + "name": "Postgres account" + } + } + }, + { + "parameters": { + "descriptionType": "manual", + "toolDescription": "Given a file ID, fetches the text from the document.", + "operation": "executeQuery", + "query": "SELECT \n string_agg(content, ' ') as document_text\nFROM documents\n WHERE metadata->>'file_id' = $1\nGROUP BY metadata->>'file_id';", + "options": { + "queryReplacement": "={{ $fromAI('file_id') }}" + } + }, + "type": "n8n-nodes-base.postgresTool", + "typeVersion": 2.5, + "position": [ + -320, + 360 + ], + "id": "7d4eb63b-3b03-4d89-ad19-9a57f1a7cd84", + "name": "Get File Contents", + "credentials": { + "postgres": { + "id": "UaTmh0frrACTMPxG", + "name": "Postgres account" + } + } + }, + { + "parameters": { + "descriptionType": "manual", + "toolDescription": "Run a SQL query - use this to query from the document_rows table once you know the file ID (which is the file path) you are querying. dataset_id is the file_id (file path) and you are always using the row_data for filtering, which is a jsonb field that has all the keys from the file schema given in the document_metadata table.\n\nExample query:\n\nSELECT AVG((row_data->>'revenue')::numeric)\nFROM document_rows\nWHERE dataset_id = '/data/shared/document.csv';\n\nExample query 2:\n\nSELECT \n row_data->>'category' as category,\n SUM((row_data->>'sales')::numeric) as total_sales\nFROM dataset_rows\nWHERE dataset_id = '/data/shared/document2.csv'\nGROUP BY row_data->>'category';", + "operation": "executeQuery", + "query": "{{ $fromAI('sql_query') }}", + "options": {} + }, + "type": "n8n-nodes-base.postgresTool", + "typeVersion": 2.5, + "position": [ + -160, + 360 + ], + "id": "07c5525b-ae30-4c1f-b3e2-a6db965c25cf", + "name": "Query Document Rows", + "credentials": { + "postgres": { + "id": "UaTmh0frrACTMPxG", + "name": "Postgres account" + } + } + }, + { + "parameters": { + "options": { + "reset": false + } + }, + "type": "n8n-nodes-base.splitInBatches", + "typeVersion": 3, + "position": [ + -1500, + 660 + ], + "id": "933204fb-4c04-4217-8045-85358e9c9b0f", + "name": "Loop Over Items" + }, + { + "parameters": { + "operation": "upsert", + "schema": { + "__rl": true, + "mode": "list", + "value": "public" + }, + "table": { + "__rl": true, + "value": "document_metadata", + "mode": "list", + "cachedResultName": "document_metadata" + }, + "columns": { + "mappingMode": "defineBelow", + "value": { + "id": "={{ $('Set File ID').item.json.file_id }}", + "title": "={{ $('Set File ID').item.json.file_title }}" + }, + "matchingColumns": [ + "id" + ], + "schema": [ + { + "id": "id", + "displayName": "id", + "required": true, + "defaultMatch": true, + "display": true, + "type": "string", + "canBeUsedToMatch": true, + "removed": false + }, + { + "id": "title", + "displayName": "title", + "required": false, + "defaultMatch": false, + "display": true, + "type": "string", + "canBeUsedToMatch": false + }, + { + "id": "url", + "displayName": "url", + "required": false, + "defaultMatch": false, + "display": true, + "type": "string", + "canBeUsedToMatch": false, + "removed": true + }, + { + "id": "created_at", + "displayName": "created_at", + "required": false, + "defaultMatch": false, + "display": true, + "type": "dateTime", + "canBeUsedToMatch": false + }, + { + "id": "schema", + "displayName": "schema", + "required": false, + "defaultMatch": false, + "display": true, + "type": "string", + "canBeUsedToMatch": false, + "removed": true + } + ], + "attemptToConvertTypes": false, + "convertFieldsToString": false + }, + "options": {} + }, + "type": "n8n-nodes-base.postgres", + "typeVersion": 2.5, + "position": [ + -820, + 680 + ], + "id": "e2be1ad8-b54c-456f-814a-6d19808b0343", + "name": "Insert Document Metadata", + "executeOnce": true, + "credentials": { + "postgres": { + "id": "UaTmh0frrACTMPxG", + "name": "Postgres account" + } + } + }, + { + "parameters": { + "schema": { + "__rl": true, + "mode": "list", + "value": "public" + }, + "table": { + "__rl": true, + "value": "document_rows", + "mode": "list", + "cachedResultName": "document_rows" + }, + "columns": { + "mappingMode": "defineBelow", + "value": { + "dataset_id": "={{ $('Set File ID').item.json.file_id }}", + "row_data": "={{ $json.toJsonString().replaceAll(/'/g, \"''\") }}" + }, + "matchingColumns": [ + "id" + ], + "schema": [ + { + "id": "id", + "displayName": "id", + "required": false, + "defaultMatch": true, + "display": true, + "type": "number", + "canBeUsedToMatch": true, + "removed": true + }, + { + "id": "dataset_id", + "displayName": "dataset_id", + "required": false, + "defaultMatch": false, + "display": true, + "type": "string", + "canBeUsedToMatch": true, + "removed": false + }, + { + "id": "row_data", + "displayName": "row_data", + "required": false, + "defaultMatch": false, + "display": true, + "type": "object", + "canBeUsedToMatch": true, + "removed": false + } + ], + "attemptToConvertTypes": false, + "convertFieldsToString": false + }, + "options": {} + }, + "type": "n8n-nodes-base.postgres", + "typeVersion": 2.5, + "position": [ + 240, + 940 + ], + "id": "be314c88-6e7d-4229-9d06-cba976ce0ca1", + "name": "Insert Table Rows", + "credentials": { + "postgres": { + "id": "UaTmh0frrACTMPxG", + "name": "Postgres account" + } + } + }, + { + "parameters": { + "operation": "upsert", + "schema": { + "__rl": true, + "mode": "list", + "value": "public" + }, + "table": { + "__rl": true, + "value": "document_metadata", + "mode": "list", + "cachedResultName": "document_metadata" + }, + "columns": { + "mappingMode": "defineBelow", + "value": { + "id": "={{ $('Set File ID').item.json.file_id }}", + "schema": "={{ $json.schema }}" + }, + "matchingColumns": [ + "id" + ], + "schema": [ + { + "id": "id", + "displayName": "id", + "required": true, + "defaultMatch": true, + "display": true, + "type": "string", + "canBeUsedToMatch": true, + "removed": false + }, + { + "id": "title", + "displayName": "title", + "required": false, + "defaultMatch": false, + "display": true, + "type": "string", + "canBeUsedToMatch": false, + "removed": true + }, + { + "id": "url", + "displayName": "url", + "required": false, + "defaultMatch": false, + "display": true, + "type": "string", + "canBeUsedToMatch": false, + "removed": true + }, + { + "id": "created_at", + "displayName": "created_at", + "required": false, + "defaultMatch": false, + "display": true, + "type": "dateTime", + "canBeUsedToMatch": false + }, + { + "id": "schema", + "displayName": "schema", + "required": false, + "defaultMatch": false, + "display": true, + "type": "string", + "canBeUsedToMatch": false, + "removed": false + } + ], + "attemptToConvertTypes": false, + "convertFieldsToString": false + }, + "options": {} + }, + "type": "n8n-nodes-base.postgres", + "typeVersion": 2.5, + "position": [ + 1100, + 700 + ], + "id": "72eb8ed6-de71-4e46-bdaf-fb8ca48225ab", + "name": "Update Schema for Document Metadata", + "credentials": { + "postgres": { + "id": "UaTmh0frrACTMPxG", + "name": "Postgres account" + } + } + }, + { + "parameters": { + "content": "## 🚀 n8n Local AI Agentic RAG Template\n\n**Author:** [Cole Medin](https://www.youtube.com/@ColeMedin)\n\n## What is this?\nThis template provides an entirely local implementation of an **Agentic RAG (Retrieval Augmented Generation)** system in n8n that can be extended easily for your specific use case and knowledge base. Unlike standard RAG which only performs simple lookups, this agent can reason about your knowledge base, self-improve retrieval, and dynamically switch between different tools based on the specific question. This workflow can be used with the [local AI package](https://github.com/coleam00/local-ai-packaged) for your LLMs (Ollama) and database (Supabase).\n\n## Why Agentic RAG?\nStandard RAG has significant limitations:\n- Poor analysis of numerical/tabular data\n- Missing context due to document chunking\n- Inability to connect information across documents\n- No dynamic tool selection based on question type\n\n## What makes this template powerful:\n- **Intelligent tool selection**: Switches between RAG lookups, SQL queries, or full document retrieval based on the question\n- **Complete document context**: Accesses entire documents when needed instead of just chunks\n- **Accurate numerical analysis**: Uses SQL for precise calculations on spreadsheet/tabular data\n- **Cross-document insights**: Connects information across your entire knowledge base\n- **Multi-file processing**: Handles multiple documents in a single workflow loop\n- **Efficient storage**: Uses JSONB in Supabase to store tabular data without creating new tables for each CSV\n\n## Getting Started\n1. Run the table creation nodes first to set up your database tables in Supabase\n2. Upload your documents to the folder on your computer that is mounted to /data/shared in the n8n container. This folder by default is the \"shared\" folder in the local AI package.\n3. The agent will process them automatically (chunking text, storing tabular data in Supabase)\n4. Start asking questions that leverage the agent's multiple reasoning approaches\n\n## Customization\nThis template provides a solid foundation that you can extend by:\n- Tuning the system prompt for your specific use case\n- Adding document metadata like summaries\n- Implementing more advanced RAG techniques\n- Optimizing for larger knowledge bases\n\n---\n\nThe non-local (\"cloud\") version of this Agentic RAG agent can be [found here](https://github.com/coleam00/ottomator-agents/tree/main/n8n-agentic-rag-agent).", + "height": 1320, + "width": 540, + "color": 6 + }, + "type": "n8n-nodes-base.stickyNote", + "position": [ + -2300, + -140 + ], + "typeVersion": 1, + "id": "69bc6648-74a5-4f24-8539-63fdc890bf8e", + "name": "Sticky Note9" + }, + { + "parameters": { + "triggerOn": "folder", + "path": "/data/shared", + "events": [ + "add", + "change" + ], + "options": { + "followSymlinks": true, + "usePolling": true + } + }, + "type": "n8n-nodes-base.localFileTrigger", + "typeVersion": 1, + "position": [ + -1700, + 660 + ], + "id": "c0039541-9e7b-487b-bf8a-4917fcc5b9dd", + "name": "Local File Trigger" + }, + { + "parameters": { + "fileSelector": "={{ $('Set File ID').item.json.file_id }}", + "options": { + "dataPropertyName": "=data" + } + }, + "type": "n8n-nodes-base.readWriteFile", + "typeVersion": 1, + "position": [ + -660, + 820 + ], + "id": "3f1993e1-77f9-4983-8eff-446f8a39d1a7", + "name": "Read/Write Files from Disk" + }, + { + "parameters": { + "model": "nomic-embed-text:latest" + }, + "type": "@n8n/n8n-nodes-langchain.embeddingsOllama", + "typeVersion": 1, + "position": [ + 760, + 1140 + ], + "id": "a6c810f0-c564-4faa-8c69-fc22ef4f1156", + "name": "Embeddings Ollama", + "credentials": { + "ollamaApi": { + "id": "GwjiKiEsG5HnTaAf", + "name": "Ollama account" + } + } + }, + { + "parameters": { + "model": "nomic-embed-text:latest" + }, + "type": "@n8n/n8n-nodes-langchain.embeddingsOllama", + "typeVersion": 1, + "position": [ + 260, + 340 + ], + "id": "31c8cf4e-525f-47e4-a499-73a0cbf4f6e5", + "name": "Embeddings Ollama1", + "credentials": { + "ollamaApi": { + "id": "GwjiKiEsG5HnTaAf", + "name": "Ollama account" + } + } + }, + { + "parameters": { + "chunkSize": 400, + "options": {} + }, + "type": "@n8n/n8n-nodes-langchain.textSplitterRecursiveCharacterTextSplitter", + "typeVersion": 1, + "position": [ + 900, + 1260 + ], + "id": "590bdbf7-cbe7-405c-b2b8-cd73155ec4e4", + "name": "Recursive Character Text Splitter" + }, + { + "parameters": { + "model": { + "__rl": true, + "value": "qwen2.5:7b-8k", + "mode": "list", + "cachedResultName": "qwen2.5:7b-8k" + }, + "options": {} + }, + "type": "@n8n/n8n-nodes-langchain.lmChatOpenAi", + "typeVersion": 1.2, + "position": [ + -740, + 360 + ], + "id": "a3b834b1-5fc7-4c84-86f3-eacde77ed46b", + "name": "Ollama (Change Base URL)", + "credentials": { + "openAiApi": { + "id": "yjDbkWOXNXC959ei", + "name": "OpenAi account" + } + } + }, + { + "parameters": { + "content": "## NOTE\n\nThe Ollama chat model node doesn't work with the RAG nodes - known issue with n8n.\n\nSo for now, we are using the OpenAI chat model but changing the base URL to Ollama when creating the credentials (i.e. http://ollama:11434/v1). The API key can be set to whatever, it isn't used for local LLMs.", + "height": 200, + "width": 540, + "color": 6 + }, + "type": "n8n-nodes-base.stickyNote", + "position": [ + -2300, + 1200 + ], + "typeVersion": 1, + "id": "c1a15e44-f282-4a2a-abe4-f4c17709fe38", + "name": "Sticky Note4" + }, + { + "parameters": { + "operation": "executeQuery", + "query": "DO $$\nBEGIN\n IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'documents_pg') THEN\n EXECUTE 'DELETE FROM documents_pg WHERE metadata->>''file_id'' LIKE ''%' || $1 || '%''';\n END IF;\nEND\n$$;", + "options": { + "queryReplacement": "={{ $json.file_id }}" + } + }, + "type": "n8n-nodes-base.postgres", + "typeVersion": 2.5, + "position": [ + -1140, + 680 + ], + "id": "b7976820-6a79-4908-861c-f9062c2f7d17", + "name": "Delete Old Doc Records", + "credentials": { + "postgres": { + "id": "UaTmh0frrACTMPxG", + "name": "Postgres account" + } + } + }, + { + "parameters": { + "operation": "executeQuery", + "query": "DELETE FROM document_rows\nWHERE dataset_id LIKE '%' || $1 || '%';", + "options": { + "queryReplacement": "={{ $('Set File ID').item.json.file_id }}" + } + }, + "type": "n8n-nodes-base.postgres", + "typeVersion": 2.5, + "position": [ + -980, + 820 + ], + "id": "979948a0-7449-46d8-8c38-0a8ff8689162", + "name": "Delete Old Data Records", + "credentials": { + "postgres": { + "id": "UaTmh0frrACTMPxG", + "name": "Postgres account" + } + } + }, + { + "parameters": { + "mode": "insert", + "tableName": "documents_pg", + "options": {} + }, + "type": "@n8n/n8n-nodes-langchain.vectorStorePGVector", + "typeVersion": 1, + "position": [ + 880, + 920 + ], + "id": "a6f5e003-5f65-488b-9ce5-0844413239f5", + "name": "Postgres PGVector Store", + "credentials": { + "postgres": { + "id": "UaTmh0frrACTMPxG", + "name": "Postgres account" + } + } + }, + { + "parameters": { + "mode": "retrieve-as-tool", + "toolName": "documents", + "toolDescription": "Use RAG to look up information in the knowledgebase.", + "tableName": "documents_pg", + "options": {} + }, + "type": "@n8n/n8n-nodes-langchain.vectorStorePGVector", + "typeVersion": 1, + "position": [ + 160, + 140 + ], + "id": "d62a3b1e-5e74-4e46-b626-fd17fb74468c", + "name": "Postgres PGVector Store1", + "credentials": { + "postgres": { + "id": "UaTmh0frrACTMPxG", + "name": "Postgres account" + } + } + } + ], + "pinData": {}, + "connections": { + "Extract Document Text": { + "main": [ + [ + { + "node": "Postgres PGVector Store", + "type": "main", + "index": 0 + } + ] + ] + }, + "Default Data Loader": { + "ai_document": [ + [ + { + "node": "Postgres PGVector Store", + "type": "ai_document", + "index": 0 + } + ] + ] + }, + "Postgres Chat Memory": { + "ai_memory": [ + [ + { + "node": "RAG AI Agent", + "type": "ai_memory", + "index": 0 + } + ] + ] + }, + "Set File ID": { + "main": [ + [ + { + "node": "Delete Old Doc Records", + "type": "main", + "index": 0 + } + ] + ] + }, + "Edit Fields": { + "main": [ + [ + { + "node": "RAG AI Agent", + "type": "main", + "index": 0 + } + ] + ] + }, + "When chat message received": { + "main": [ + [ + { + "node": "Edit Fields", + "type": "main", + "index": 0 + } + ] + ] + }, + "Webhook": { + "main": [ + [ + { + "node": "Edit Fields", + "type": "main", + "index": 0 + } + ] + ] + }, + "Extract PDF Text": { + "main": [ + [ + { + "node": "Postgres PGVector Store", + "type": "main", + "index": 0 + } + ] + ] + }, + "Aggregate": { + "main": [ + [ + { + "node": "Summarize", + "type": "main", + "index": 0 + } + ] + ] + }, + "Summarize": { + "main": [ + [ + { + "node": "Set Schema", + "type": "main", + "index": 0 + }, + { + "node": "Postgres PGVector Store", + "type": "main", + "index": 0 + } + ] + ] + }, + "RAG AI Agent": { + "main": [ + [ + { + "node": "Respond to Webhook", + "type": "main", + "index": 0 + } + ] + ] + }, + "Switch": { + "main": [ + [ + { + "node": "Extract PDF Text", + "type": "main", + "index": 0 + } + ], + [ + { + "node": "Extract from Excel", + "type": "main", + "index": 0 + } + ], + [ + { + "node": "Extract from CSV", + "type": "main", + "index": 0 + } + ], + [ + { + "node": "Extract Document Text", + "type": "main", + "index": 0 + } + ] + ] + }, + "Extract from Excel": { + "main": [ + [ + { + "node": "Aggregate", + "type": "main", + "index": 0 + }, + { + "node": "Insert Table Rows", + "type": "main", + "index": 0 + } + ] + ] + }, + "Set Schema": { + "main": [ + [ + { + "node": "Update Schema for Document Metadata", + "type": "main", + "index": 0 + } + ] + ] + }, + "Extract from CSV": { + "main": [ + [ + { + "node": "Aggregate", + "type": "main", + "index": 0 + }, + { + "node": "Insert Table Rows", + "type": "main", + "index": 0 + } + ] + ] + }, + "Create Document Metadata Table": { + "main": [ + [] + ] + }, + "List Documents": { + "ai_tool": [ + [ + { + "node": "RAG AI Agent", + "type": "ai_tool", + "index": 0 + } + ] + ] + }, + "Get File Contents": { + "ai_tool": [ + [ + { + "node": "RAG AI Agent", + "type": "ai_tool", + "index": 0 + } + ] + ] + }, + "Query Document Rows": { + "ai_tool": [ + [ + { + "node": "RAG AI Agent", + "type": "ai_tool", + "index": 0 + } + ] + ] + }, + "Loop Over Items": { + "main": [ + [], + [ + { + "node": "Set File ID", + "type": "main", + "index": 0 + } + ] + ] + }, + "Insert Document Metadata": { + "main": [ + [ + { + "node": "Read/Write Files from Disk", + "type": "main", + "index": 0 + } + ] + ] + }, + "Local File Trigger": { + "main": [ + [ + { + "node": "Loop Over Items", + "type": "main", + "index": 0 + } + ] + ] + }, + "Read/Write Files from Disk": { + "main": [ + [ + { + "node": "Switch", + "type": "main", + "index": 0 + } + ] + ] + }, + "Embeddings Ollama": { + "ai_embedding": [ + [ + { + "node": "Postgres PGVector Store", + "type": "ai_embedding", + "index": 0 + } + ] + ] + }, + "Embeddings Ollama1": { + "ai_embedding": [ + [ + { + "node": "Postgres PGVector Store1", + "type": "ai_embedding", + "index": 0 + } + ] + ] + }, + "Recursive Character Text Splitter": { + "ai_textSplitter": [ + [ + { + "node": "Default Data Loader", + "type": "ai_textSplitter", + "index": 0 + } + ] + ] + }, + "Ollama (Change Base URL)": { + "ai_languageModel": [ + [ + { + "node": "RAG AI Agent", + "type": "ai_languageModel", + "index": 0 + } + ] + ] + }, + "Delete Old Doc Records": { + "main": [ + [ + { + "node": "Delete Old Data Records", + "type": "main", + "index": 0 + } + ] + ] + }, + "Delete Old Data Records": { + "main": [ + [ + { + "node": "Insert Document Metadata", + "type": "main", + "index": 0 + } + ] + ] + }, + "Postgres PGVector Store": { + "main": [ + [ + { + "node": "Loop Over Items", + "type": "main", + "index": 0 + } + ] + ] + }, + "Postgres PGVector Store1": { + "ai_tool": [ + [ + { + "node": "RAG AI Agent", + "type": "ai_tool", + "index": 0 + } + ] + ] + } + }, + "active": true, + "settings": { + "executionOrder": "v1" + }, + "versionId": "0bab9394-1560-49a0-a8c3-1ec43a250b1b", + "meta": { + "templateCredsSetupCompleted": true, + "instanceId": "73cb7a3e883df514bb47e8d1b34526d30e2abb8f56cd99f10d5948a1e11b25aa" + }, + "id": "RssROpqkXOm23GYL", + "tags": [] +} \ No newline at end of file