From 5302560a1336c9f230efa736a331e05ff9eb1aaa Mon Sep 17 00:00:00 2001 From: STetsing <41009393+STetsing@users.noreply.github.com> Date: Mon, 3 Feb 2025 14:32:08 +0100 Subject: [PATCH] added max token support && event support for indexing --- .../src/app/plugins/remixAIPlugin.tsx | 25 +++--- .../src/agents/completionAgent.ts | 79 +++++++++++++++++-- .../src/inferencers/remote/remoteInference.ts | 2 +- 3 files changed, 86 insertions(+), 20 deletions(-) diff --git a/apps/remix-ide/src/app/plugins/remixAIPlugin.tsx b/apps/remix-ide/src/app/plugins/remixAIPlugin.tsx index 14ba934f3b..312fab4f76 100644 --- a/apps/remix-ide/src/app/plugins/remixAIPlugin.tsx +++ b/apps/remix-ide/src/app/plugins/remixAIPlugin.tsx @@ -65,13 +65,6 @@ export class RemixAIPlugin extends ViewPlugin { this.initialize() } this.completionAgent = new CodeCompletionAgent(this) - - setInterval(() => { - console.log('Indexing workspace') - this.completionAgent.indexWorkspace() - }, 60000) - - this.securityAgent = new SecurityAgent(this) } @@ -119,11 +112,13 @@ export class RemixAIPlugin extends ViewPlugin { if (this.completionAgent.indexer == null || this.completionAgent.indexer == undefined) await this.completionAgent.indexWorkspace() const currentFile = await this.call('fileManager', 'getCurrentFile') - const contextfiles = await this.completionAgent.getContextFiles() + const contextfiles = await this.completionAgent.getContextFiles(prompt) + const refined = CodeCompletionAgent.refineContext(prompt, promptAfter) + console.log('contextfiles', contextfiles) if (this.isOnDesktop && !this.useRemoteInferencer) { - return await this.call(this.remixDesktopPluginName, 'code_completion', prompt, promptAfter) + return await this.call(this.remixDesktopPluginName, 'code_completion', refined[0], refined[1]) } else { - return await this.remoteInferencer.code_completion(prompt, promptAfter, contextfiles, currentFile) + return await this.remoteInferencer.code_completion(refined[0], refined[1], contextfiles, currentFile) } } @@ -185,11 +180,13 @@ export class RemixAIPlugin extends ViewPlugin { if (this.completionAgent.indexer == null || this.completionAgent.indexer == undefined) await this.completionAgent.indexWorkspace() const currentFile = await this.call('fileManager', 'getCurrentFile') - const contextfiles = this.completionAgent.getContextFiles() + const contextfiles = this.completionAgent.getContextFiles(msg_pfx) + const refined = CodeCompletionAgent.refineContext(msg_pfx, msg_sfx) + if (this.isOnDesktop && !this.useRemoteInferencer) { - return await this.call(this.remixDesktopPluginName, 'code_insertion', msg_pfx, msg_sfx) + return await this.call(this.remixDesktopPluginName, 'code_insertion', refined[0], refined[1]) } else { - return await this.remoteInferencer.code_insertion(msg_pfx, msg_sfx, contextfiles, currentFile) + return await this.remoteInferencer.code_insertion(refined[0], refined[1], contextfiles, currentFile) } } @@ -212,7 +209,7 @@ export class RemixAIPlugin extends ViewPlugin { else { console.log("chatRequestBuffer is not empty. First process the last request.", this.chatRequestBuffer) } - _paq.push(['trackEvent', 'ai', 'remixAI_chat', 'askFromTerminal']) + _paq.push(['trackEvent', 'ai', 'remixAI', 'remixAI_chat']) } async ProcessChatRequestBuffer(params:IParams=GenerationParams){ diff --git a/libs/remix-ai-core/src/agents/completionAgent.ts b/libs/remix-ai-core/src/agents/completionAgent.ts index 11dcb903cd..c31ff6a5be 100644 --- a/libs/remix-ai-core/src/agents/completionAgent.ts +++ b/libs/remix-ai-core/src/agents/completionAgent.ts @@ -7,6 +7,12 @@ interface Document { identifier: number; } +interface indexT{ + isIndexed: boolean; + lastIndexedTime?: number; + reason?: string; +} + enum SupportedFileExtensions { solidity = '.sol', vyper = '.vy', @@ -20,17 +26,36 @@ export class CodeCompletionAgent { Documents: Document[] = []; INDEX_THRESHOLD = 0.1; N_MATCHES = 1; + indexed: indexT = { + isIndexed: false, + lastIndexedTime: 0, + reason: 'Init', + }; constructor(props) { this.props = props; - this.props.on('fileManager', 'fileAdded', (path) => { }); - this.props.on('filePanel', 'workspaceCreated', async () => { }); + this.listenForChanges(); this.indexer =lunr(function () { this.ref('id') this.field('filename') this.field('content') this.field('Identifier'); }); + + // listen for file changes + this.props.on('fileManager', 'fileRemoved', (path) => { this.indexed.isIndexed = false; }); + + setInterval(() => { + this.indexWorkspace() + }, 60000) + } + + listenForChanges() { + this.props.on('fileManager', 'fileAdded', (path) => { this.indexed = { isIndexed: false, reason:"fileAdded" } }); + this.props.on('fileManager', 'fileRemoved', (path) => { this.indexed = { isIndexed: false, reason:"fileRemoved" } }); + this.props.on('filePanel', 'workspaceCreated', () => { this.indexed = { isIndexed: false, reason:"workspaceCreated" } }); + this.props.on('filePanel', 'workspaceRenamed', () => { this.indexed = { isIndexed: false, reason:"workspaceRenamed" }}); + this.props.on('filePanel', 'workspaceDeleted', () => { this.indexed = { isIndexed: false, reason:"workspaceDeleted" } }); } async getDcocuments() { @@ -56,7 +81,6 @@ export class CodeCompletionAgent { indexWorkspace() { this.getDcocuments().then((documents) => { - this.Documents = documents; this.indexer =lunr(function () { this.ref('id') this.field('filename') @@ -67,13 +91,19 @@ export class CodeCompletionAgent { this.add(doc); }); }); + this.Documents = documents; }); + + this.indexed = { isIndexed: true, lastIndexedTime: Date.now(), reason: 'init Indexing' }; } - async getContextFiles() { + async getContextFiles(prompt) { try { + if (!this.indexed.isIndexed) { + this.indexWorkspace(); + } const currentFile = await this.props.call('fileManager', 'getCurrentFile'); - const content = await this.props.call('fileManager', 'readFile', currentFile); + const content = prompt; const searchResult = this.indexer.search(content) const fcps = await this.processResults(searchResult, currentFile); const resolvedFcps = await Promise.all(fcps); @@ -100,4 +130,43 @@ export class CodeCompletionAgent { return fileContentPairs; } + // TODO rm context files length + static refineContext(words1: string, words2: string, maxWords: number=6500) { + // Avoid model out of context + const totalWords = words1.length + words2.length; + if (totalWords <= maxWords) { + console.log('total words less than max words', totalWords); + return [words1, words2]; + } + const halfMaxWords = Math.floor(maxWords / 2); + + let takeFromText1 = words1.length < halfMaxWords ? words1.length : halfMaxWords; + let takeFromText2 = words2.length < halfMaxWords ? words2.length : halfMaxWords; + + // Adjust if one text is taking more than half, we balance by limiting the other text + if (words1.length < halfMaxWords && words2.length + words1.length <= maxWords) { + takeFromText2 = Math.min(words2.length, maxWords - words1.length); + } else if (words2.length < halfMaxWords && words1.length + words2.length <= maxWords) { + takeFromText1 = Math.min(words1.length, maxWords - words2.length); + } else if (words1.length < halfMaxWords && words2.length + words1.length >= maxWords) { + takeFromText2 = Math.min(words2.length, maxWords - words1.length); + } else if (words2.length > halfMaxWords && words1.length + words2.length <= maxWords) { + takeFromText1 = Math.min(words1.length, maxWords - words2.length); + } + + const splicedText1 = words1.slice(words1.length - takeFromText1); + const splicedText2 = words2.slice(words2.length - takeFromText2); + console.log('take from text 1', takeFromText1) + console.log('take from text 2', takeFromText2) + + console.log('Spliced text 1 length:', splicedText1.length); + console.log('Spliced text 2 length:', splicedText2.length); + console.log('initial word 1 length:', words1.length); + console.log('initial word 2 length:', words2.length); + console.log('text 1:', splicedText1); + console.log('text 2:', splicedText2); + + return [splicedText1 , splicedText2] + } + } diff --git a/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts b/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts index a8fa702fc7..92eb0aea6a 100644 --- a/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts +++ b/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts @@ -12,7 +12,7 @@ export class RemoteInferencer implements ICompletions { max_history = 7 model_op = RemoteBackendOPModel.CODELLAMA // default model operation change this to llama if necessary event: EventEmitter - test_env=false + test_env=true test_url="http://solcodertest.org" constructor(apiUrl?:string, completionUrl?:string) {