added max token support && event support for indexing

apps/remix-ide/src/app/plugins/remixAIPlugin.tsx

@@ -65,13 +65,6 @@ export class RemixAIPlugin extends ViewPlugin {
       this.initialize()
     }
     this.completionAgent = new CodeCompletionAgent(this)
-
-    setInterval(() => {
-      console.log('Indexing workspace')
-      this.completionAgent.indexWorkspace()
-    }, 60000)
-
-
     this.securityAgent = new SecurityAgent(this)
   }
 
@@ -119,11 +112,13 @@ export class RemixAIPlugin extends ViewPlugin {
     if (this.completionAgent.indexer == null || this.completionAgent.indexer == undefined) await this.completionAgent.indexWorkspace()
 
     const currentFile = await this.call('fileManager', 'getCurrentFile')
-    const contextfiles = await this.completionAgent.getContextFiles()
+    const contextfiles = await this.completionAgent.getContextFiles(prompt)
+    const refined = CodeCompletionAgent.refineContext(prompt, promptAfter)
+    console.log('contextfiles', contextfiles)
     if (this.isOnDesktop && !this.useRemoteInferencer) {
-      return await this.call(this.remixDesktopPluginName, 'code_completion', prompt, promptAfter)
+      return await this.call(this.remixDesktopPluginName, 'code_completion', refined[0], refined[1])
     } else {
-      return await this.remoteInferencer.code_completion(prompt, promptAfter, contextfiles, currentFile)
+      return await this.remoteInferencer.code_completion(refined[0], refined[1], contextfiles, currentFile)
     }
   }
 
@@ -185,11 +180,13 @@ export class RemixAIPlugin extends ViewPlugin {
     if (this.completionAgent.indexer == null || this.completionAgent.indexer == undefined) await this.completionAgent.indexWorkspace()
 
     const currentFile = await this.call('fileManager', 'getCurrentFile')
-    const contextfiles = this.completionAgent.getContextFiles()
+    const contextfiles = this.completionAgent.getContextFiles(msg_pfx)
+    const refined = CodeCompletionAgent.refineContext(msg_pfx, msg_sfx)
+
     if (this.isOnDesktop && !this.useRemoteInferencer) {
-      return await this.call(this.remixDesktopPluginName, 'code_insertion', msg_pfx, msg_sfx)
+      return await this.call(this.remixDesktopPluginName, 'code_insertion', refined[0], refined[1])
     } else {
-      return await this.remoteInferencer.code_insertion(msg_pfx, msg_sfx, contextfiles, currentFile)
+      return await this.remoteInferencer.code_insertion(refined[0], refined[1], contextfiles, currentFile)
     }
   }
 
@@ -212,7 +209,7 @@ export class RemixAIPlugin extends ViewPlugin {
     else {
       console.log("chatRequestBuffer is not empty. First process the last request.", this.chatRequestBuffer)
     }
-    _paq.push(['trackEvent', 'ai', 'remixAI_chat', 'askFromTerminal'])
+    _paq.push(['trackEvent', 'ai', 'remixAI', 'remixAI_chat'])
   }
 
   async ProcessChatRequestBuffer(params:IParams=GenerationParams){

libs/remix-ai-core/src/agents/completionAgent.ts

@@ -7,6 +7,12 @@ interface Document {
   identifier: number;
 }
 
+interface indexT{
+  isIndexed: boolean;
+  lastIndexedTime?: number;
+  reason?: string;
+}
+
 enum SupportedFileExtensions {
   solidity = '.sol',
   vyper = '.vy',
@@ -20,17 +26,36 @@ export class CodeCompletionAgent {
   Documents: Document[] = [];
   INDEX_THRESHOLD = 0.1;
   N_MATCHES = 1;
+  indexed: indexT = {
+    isIndexed: false,
+    lastIndexedTime: 0,
+    reason: 'Init',
+  };
 
   constructor(props) {
     this.props = props;
-    this.props.on('fileManager', 'fileAdded', (path) => { });
+    this.listenForChanges();
-    this.props.on('filePanel', 'workspaceCreated', async () => { });
     this.indexer =lunr(function () {
       this.ref('id')
       this.field('filename')
       this.field('content')
       this.field('Identifier');
     });
+
+    // listen for file changes
+    this.props.on('fileManager', 'fileRemoved', (path) => { this.indexed.isIndexed = false; });
+
+    setInterval(() => {
+      this.indexWorkspace()
+    }, 60000)
+  }
+
+  listenForChanges() {
+    this.props.on('fileManager', 'fileAdded', (path) => { this.indexed = { isIndexed: false, reason:"fileAdded" } });
+    this.props.on('fileManager', 'fileRemoved', (path) => { this.indexed = { isIndexed: false, reason:"fileRemoved" } });
+    this.props.on('filePanel', 'workspaceCreated', () => { this.indexed = { isIndexed: false, reason:"workspaceCreated" } });
+    this.props.on('filePanel', 'workspaceRenamed', () => { this.indexed = { isIndexed: false, reason:"workspaceRenamed" }});
+    this.props.on('filePanel', 'workspaceDeleted', () => { this.indexed = { isIndexed: false, reason:"workspaceDeleted" } });
   }
 
   async getDcocuments() {
@@ -56,7 +81,6 @@ export class CodeCompletionAgent {
 
   indexWorkspace() {
     this.getDcocuments().then((documents) => {
-      this.Documents = documents;
       this.indexer =lunr(function () {
         this.ref('id')
         this.field('filename')
@@ -67,13 +91,19 @@ export class CodeCompletionAgent {
           this.add(doc);
         });
       });
+      this.Documents = documents;
     });
+
+    this.indexed = { isIndexed: true, lastIndexedTime: Date.now(), reason: 'init Indexing' };
   }
 
-  async getContextFiles() {
+  async getContextFiles(prompt) {
     try {
+      if (!this.indexed.isIndexed) {
+        this.indexWorkspace();
+      }
       const currentFile = await this.props.call('fileManager', 'getCurrentFile');
-      const content = await this.props.call('fileManager', 'readFile', currentFile);
+      const content = prompt;
       const searchResult = this.indexer.search(content)
       const fcps = await this.processResults(searchResult, currentFile);
       const resolvedFcps = await Promise.all(fcps);
@@ -100,4 +130,43 @@ export class CodeCompletionAgent {
     return fileContentPairs;
   }
 
+  // TODO rm context files length
+  static refineContext(words1: string, words2: string, maxWords: number=6500) {
+    // Avoid model out of context
+    const totalWords = words1.length + words2.length;
+    if (totalWords <= maxWords) {
+      console.log('total words less than max words', totalWords);
+      return [words1, words2];
+    }
+    const halfMaxWords = Math.floor(maxWords / 2);
+
+    let takeFromText1 = words1.length < halfMaxWords ? words1.length : halfMaxWords;
+    let takeFromText2 = words2.length < halfMaxWords ? words2.length : halfMaxWords;
+
+    // Adjust if one text is taking more than half, we balance by limiting the other text
+    if (words1.length < halfMaxWords && words2.length + words1.length <= maxWords) {
+      takeFromText2 = Math.min(words2.length, maxWords - words1.length);
+    } else if (words2.length < halfMaxWords && words1.length + words2.length <= maxWords) {
+      takeFromText1 = Math.min(words1.length, maxWords - words2.length);
+    } else if (words1.length < halfMaxWords && words2.length + words1.length >= maxWords) {
+      takeFromText2 = Math.min(words2.length, maxWords - words1.length);
+    } else if (words2.length > halfMaxWords && words1.length + words2.length <= maxWords) {
+      takeFromText1 = Math.min(words1.length, maxWords - words2.length);
+    }
+
+    const splicedText1 = words1.slice(words1.length - takeFromText1);
+    const splicedText2 = words2.slice(words2.length - takeFromText2);
+    console.log('take from text 1', takeFromText1)
+    console.log('take from text 2', takeFromText2)
+
+    console.log('Spliced text 1 length:', splicedText1.length);
+    console.log('Spliced text 2 length:', splicedText2.length);
+    console.log('initial word 1 length:', words1.length);
+    console.log('initial word 2 length:', words2.length);
+    console.log('text 1:', splicedText1);
+    console.log('text 2:', splicedText2);
+
+    return [splicedText1 , splicedText2]
+  }
+
 }

libs/remix-ai-core/src/inferencers/remote/remoteInference.ts

@@ -12,7 +12,7 @@ export class RemoteInferencer implements ICompletions {
   max_history = 7
   model_op = RemoteBackendOPModel.CODELLAMA // default model operation change this to llama if necessary
   event: EventEmitter
-  test_env=false
+  test_env=true
   test_url="http://solcodertest.org"
 
   constructor(apiUrl?:string, completionUrl?:string) {