From fab2076eb30048943cf7240b9f55876e83b99478 Mon Sep 17 00:00:00 2001
From: STetsing <41009393+STetsing@users.noreply.github.com>
Date: Mon, 20 Jan 2025 14:18:52 +0100
Subject: [PATCH 01/16] initial impl

---
 .../src/app/plugins/remixAIPlugin.tsx         |  5 ++
 apps/vyper/src/app/utils/remix-client.tsx     |  1 +
 .../src/agents/completionAgent.ts             | 85 +++++++++++++++----
 libs/remix-ai-core/src/index.ts               |  3 +-
 .../src/lib/components/dynamicButtons.tsx     | 22 +++++
 package.json                                  |  2 +
 yarn.lock                                     | 10 +++
 7 files changed, 112 insertions(+), 16 deletions(-)
 create mode 100644 libs/remix-ui/remix-ai/src/lib/components/dynamicButtons.tsx

diff --git a/apps/remix-ide/src/app/plugins/remixAIPlugin.tsx b/apps/remix-ide/src/app/plugins/remixAIPlugin.tsx
index dbe2dcb4e5..f919425bf4 100644
--- a/apps/remix-ide/src/app/plugins/remixAIPlugin.tsx
+++ b/apps/remix-ide/src/app/plugins/remixAIPlugin.tsx
@@ -6,6 +6,7 @@ import React, { useCallback } from 'react';
 import { ICompletions, IModel, RemoteInferencer, IRemoteModel, IParams, GenerationParams, CodeExplainAgent } from '@remix/remix-ai-core';
 import { CustomRemixApi } from '@remix-api'
 import { PluginViewWrapper } from '@remix-ui/helper'
+import { CodeCompletionAgent } from '@remix/remix-ai-core';
 const _paq = (window._paq = window._paq || [])
 
 type chatRequestBufferT<T> = {
@@ -41,6 +42,7 @@ export class RemixAIPlugin extends ViewPlugin {
   agent: CodeExplainAgent
   useRemoteInferencer:boolean = false
   dispatch: any
+  completionAgent: CodeCompletionAgent
 
   constructor(inDesktop:boolean) {
     super(profile)
@@ -62,6 +64,7 @@ export class RemixAIPlugin extends ViewPlugin {
       this.useRemoteInferencer = true
       this.initialize()
     }
+    this.completionAgent = new CodeCompletionAgent(this)
   }
 
   async initialize(model1?:IModel, model2?:IModel, remoteModel?:IRemoteModel, useRemote?:boolean){
@@ -110,6 +113,7 @@ export class RemixAIPlugin extends ViewPlugin {
   }
 
   async code_completion(prompt: string, promptAfter: string): Promise<any> {
+    this.completionAgent.searchIndex(prompt)
     if (this.isOnDesktop && !this.useRemoteInferencer) {
       return await this.call(this.remixDesktopPluginName, 'code_completion', prompt, promptAfter)
     } else {
@@ -175,6 +179,7 @@ export class RemixAIPlugin extends ViewPlugin {
   }
 
   async code_insertion(msg_pfx: string, msg_sfx: string): Promise<any> {
+    this.completionAgent.indexWorkspace()
     if (this.isOnDesktop && !this.useRemoteInferencer) {
       return await this.call(this.remixDesktopPluginName, 'code_insertion', msg_pfx, msg_sfx)
     } else {
diff --git a/apps/vyper/src/app/utils/remix-client.tsx b/apps/vyper/src/app/utils/remix-client.tsx
index a52e34ab8a..fb09e559c8 100644
--- a/apps/vyper/src/app/utils/remix-client.tsx
+++ b/apps/vyper/src/app/utils/remix-client.tsx
@@ -69,6 +69,7 @@ export class RemixClient extends PluginClient<any, CustomRemixApi> {
     try {
       // TODO: remove! no formatting required since already handled on server
       const file = await this.client.call('fileManager', 'getCurrentFile')
+      
       const content = await this.client.call('fileManager', 'readFile', file)
       const messageAI = `Vyper code: ${content}\n error message: ${message}\n explain why the error occurred and how to fix it.`
 
diff --git a/libs/remix-ai-core/src/agents/completionAgent.ts b/libs/remix-ai-core/src/agents/completionAgent.ts
index 1bf14005c7..42252e99ec 100644
--- a/libs/remix-ai-core/src/agents/completionAgent.ts
+++ b/libs/remix-ai-core/src/agents/completionAgent.ts
@@ -1,23 +1,78 @@
-import * as fs from 'fs';
+import FlexSearch from 'flexsearch';
+import lunr from 'lunr';
 
-class CodeCompletionAgent {
-  private codebase: string[];
+interface Document {
+  id: number;
+  filename: string;
+  content: string;
+  identifier: number;
+}
+
+enum SupportedFileExtensions {
+  solidity = '.sol',
+  vyper = '.vy',
+  circom = '.circom',
+}
+
+export class CodeCompletionAgent {
+  private workspacesIndexes = new Map<string, any>();
+  props: any;
+  indexer: any;
+  Documents: Document[] = [];
 
-  constructor(codebasePath: string) {
-    // git or fs
-    this.codebase = this.loadCodebase(codebasePath);
+  constructor(props) {
+    this.props = props;
+    this.props.on('fileManager', 'fileAdded', (path) => { });
+    this.props.on('filePanel', 'workspaceCreated', async () => { });
   }
 
-  private loadCodebase(path: string): string[] {
-    const files = fs.readdirSync(path);
-    return files
-      .filter(file => file.endsWith('.ts'))
-      .flatMap(file => fs.readFileSync(`${path}/${file}`, 'utf-8').split('\n'));
+  async getDcocuments() {
+    const documents: Document[] = [];
+    const dirs = await this.props.call('fileManager', 'dirList', '/');
+    let c = 0;
+    for (const dir of dirs) {
+      const files = await this.props.call('fileManager', 'fileList', dir);
+      for (const file of files) {
+        const content = await this.props.call('fileManager', 'readFile', file);
+        // filter out any  SupportedFileExtensions
+        if (! Object.values(SupportedFileExtensions).some(ext => file.endsWith(ext))) continue;
+        documents.push({
+          id: ++c,
+          filename: file,
+          content: content,
+          identifier: c*34,
+        });
+      }
+    }
+    console.log('Documents', documents);
+    return documents;
   }
 
-  public getSuggestions(currentLine: string, numSuggestions: number = 3): string[] {
-    const suggestions: string[] = [];
-    // get `numSuggestions` from the llm
-    return suggestions;
+  indexWorkspace() {
+    this.getDcocuments().then((documents) => {
+      this.Documents = documents;
+      this.indexer =lunr(function () {
+        this.ref('id')
+        this.field('filename')
+        this.field('content')
+        this.field('Identifier');
+
+        documents.forEach(doc => {
+          this.add(doc);
+        });
+      });
+    });
+  }
+
+  async searchIndex(query: string) {
+    try {
+      const searchResult = this.indexer.search(query);
+      console.log('Search result', searchResult);
+      return searchResult[0];
+    } catch (error) {
+      console.log('Error searching index', error);
+      return null;
+    }
   }
+
 }
diff --git a/libs/remix-ai-core/src/index.ts b/libs/remix-ai-core/src/index.ts
index fe54a57f2f..1b06d57138 100644
--- a/libs/remix-ai-core/src/index.ts
+++ b/libs/remix-ai-core/src/index.ts
@@ -21,4 +21,5 @@ export {
 
 export * from './types/types'
 export * from './helpers/streamHandler'
-export * from './agents/codeExplainAgent'
\ No newline at end of file
+export * from './agents/codeExplainAgent'
+export * from './agents/completionAgent'
\ No newline at end of file
diff --git a/libs/remix-ui/remix-ai/src/lib/components/dynamicButtons.tsx b/libs/remix-ui/remix-ai/src/lib/components/dynamicButtons.tsx
new file mode 100644
index 0000000000..181e4d3247
--- /dev/null
+++ b/libs/remix-ui/remix-ai/src/lib/components/dynamicButtons.tsx
@@ -0,0 +1,22 @@
+import React from 'react';
+
+interface ButtonProps {
+  label: string;
+  onClick: () => void;
+  icon?: string;
+}
+
+const DynamicButtons: React.FC<{ buttons: ButtonProps[] }> = ({ buttons }) => {
+  return (
+    <div className="dynamic-buttons">
+      {buttons.map((button, index) => (
+        <button key={index} onClick={button.onClick}>
+          {button.icon && <i className={button.icon}></i>}
+          {button.label}
+        </button>
+      ))}
+    </div>
+  );
+};
+
+export default DynamicButtons;
\ No newline at end of file
diff --git a/package.json b/package.json
index b34d102117..e231703e96 100644
--- a/package.json
+++ b/package.json
@@ -153,6 +153,7 @@
     "express-ws": "^5.0.2",
     "file-path-filter": "^3.0.2",
     "file-saver": "^2.0.5",
+    "flexsearch": "^0.7.43",
     "form-data": "^4.0.0",
     "formik": "^2.4.5",
     "from-exponential": "1.1.1",
@@ -171,6 +172,7 @@
     "jszip": "^3.6.0",
     "just-once": "^2.2.0",
     "latest-version": "^5.1.0",
+    "lunr": "^2.3.9",
     "merge": "^2.1.1",
     "npm-install-version": "^6.0.2",
     "octokit": "^3.1.2",
diff --git a/yarn.lock b/yarn.lock
index 89c2ba8cd6..57ae2d08ae 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -16080,6 +16080,11 @@ flatted@^3.1.0:
   resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.2.7.tgz#609f39207cb614b89d0765b477cb2d437fbf9787"
   integrity sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==
 
+flexsearch@^0.7.43:
+  version "0.7.43"
+  resolved "https://registry.yarnpkg.com/flexsearch/-/flexsearch-0.7.43.tgz#34f89b36278a466ce379c5bf6fb341965ed3f16c"
+  integrity sha512-c5o/+Um8aqCSOXGcZoqZOm+NqtVwNsvVpWv6lfmSclU954O3wvQKxxK8zj74fPaSJbXpSLTs4PRhh+wnoCXnKg==
+
 flora-colossus@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/flora-colossus/-/flora-colossus-1.0.1.tgz#aba198425a8185341e64f9d2a6a96fd9a3cbdb93"
@@ -21348,6 +21353,11 @@ lru_map@^0.3.3:
   resolved "https://registry.yarnpkg.com/lru_map/-/lru_map-0.3.3.tgz#b5c8351b9464cbd750335a79650a0ec0e56118dd"
   integrity sha512-Pn9cox5CsMYngeDbmChANltQl+5pi6XmTrraMSzhPmMBbmgcxmqWry0U3PGapCU1yB4/LqCcom7qhHZiF/jGfQ==
 
+lunr@^2.3.9:
+  version "2.3.9"
+  resolved "https://registry.yarnpkg.com/lunr/-/lunr-2.3.9.tgz#18b123142832337dd6e964df1a5a7707b25d35e1"
+  integrity sha512-zTU3DaZaF3Rt9rhN3uBMGQD3dD2/vFQqnvZCDv4dl5iOzq2IZQqTxu90r4E5J+nP70J3ilqVCrbho2eWaeW8Ow==
+
 lz-string@^1.4.4:
   version "1.4.4"
   resolved "https://registry.yarnpkg.com/lz-string/-/lz-string-1.4.4.tgz#c0d8eaf36059f705796e1e344811cf4c498d3a26"

From 7723356d6664b0244749227220b0b2ff5ea2f2ee Mon Sep 17 00:00:00 2001
From: STetsing <41009393+STetsing@users.noreply.github.com>
Date: Fri, 24 Jan 2025 15:54:42 +0100
Subject: [PATCH 02/16] initial working workspace completion

---
 .../src/app/plugins/remixAIPlugin.tsx         | 21 ++++++++---
 .../src/agents/completionAgent.ts             | 35 ++++++++++++++-----
 .../src/inferencers/remote/remoteInference.ts | 13 ++++---
 libs/remix-ai-core/src/types/types.ts         |  4 +--
 4 files changed, 54 insertions(+), 19 deletions(-)

diff --git a/apps/remix-ide/src/app/plugins/remixAIPlugin.tsx b/apps/remix-ide/src/app/plugins/remixAIPlugin.tsx
index f919425bf4..69c415c1ed 100644
--- a/apps/remix-ide/src/app/plugins/remixAIPlugin.tsx
+++ b/apps/remix-ide/src/app/plugins/remixAIPlugin.tsx
@@ -65,6 +65,12 @@ export class RemixAIPlugin extends ViewPlugin {
       this.initialize()
     }
     this.completionAgent = new CodeCompletionAgent(this)
+
+    // each 10 seconds update the workspace index
+    setInterval(() => {
+      console.log('Indexing workspace')
+      this.completionAgent.indexWorkspace()
+    }, 60000)
   }
 
   async initialize(model1?:IModel, model2?:IModel, remoteModel?:IRemoteModel, useRemote?:boolean){
@@ -113,11 +119,15 @@ export class RemixAIPlugin extends ViewPlugin {
   }
 
   async code_completion(prompt: string, promptAfter: string): Promise<any> {
-    this.completionAgent.searchIndex(prompt)
+    if (this.completionAgent.indexer == null || this.completionAgent.indexer == undefined) await this.completionAgent.indexWorkspace()
+
+    const currentFile = await this.call('fileManager', 'getCurrentFile')
+    const contextfiles = await this.completionAgent.getContextFiles()
+    console.log('completion Context files', contextfiles)
     if (this.isOnDesktop && !this.useRemoteInferencer) {
       return await this.call(this.remixDesktopPluginName, 'code_completion', prompt, promptAfter)
     } else {
-      return await this.remoteInferencer.code_completion(prompt, promptAfter)
+      return await this.remoteInferencer.code_completion(prompt, promptAfter, contextfiles, currentFile)
     }
   }
 
@@ -179,11 +189,14 @@ export class RemixAIPlugin extends ViewPlugin {
   }
 
   async code_insertion(msg_pfx: string, msg_sfx: string): Promise<any> {
-    this.completionAgent.indexWorkspace()
+    if (this.completionAgent.indexer == null || this.completionAgent.indexer == undefined) await this.completionAgent.indexWorkspace()
+
+    const currentFile = await this.call('fileManager', 'getCurrentFile')
+    const contextfiles = this.completionAgent.getContextFiles()
     if (this.isOnDesktop && !this.useRemoteInferencer) {
       return await this.call(this.remixDesktopPluginName, 'code_insertion', msg_pfx, msg_sfx)
     } else {
-      return await this.remoteInferencer.code_insertion(msg_pfx, msg_sfx)
+      return await this.remoteInferencer.code_insertion(msg_pfx, msg_sfx, contextfiles, currentFile)
     }
   }
 
diff --git a/libs/remix-ai-core/src/agents/completionAgent.ts b/libs/remix-ai-core/src/agents/completionAgent.ts
index 42252e99ec..dfc1a44493 100644
--- a/libs/remix-ai-core/src/agents/completionAgent.ts
+++ b/libs/remix-ai-core/src/agents/completionAgent.ts
@@ -1,4 +1,3 @@
-import FlexSearch from 'flexsearch';
 import lunr from 'lunr';
 
 interface Document {
@@ -19,6 +18,8 @@ export class CodeCompletionAgent {
   props: any;
   indexer: any;
   Documents: Document[] = [];
+  INDEX_THRESHOLD = 0.1;
+  N_MATCHES = 1;
 
   constructor(props) {
     this.props = props;
@@ -40,7 +41,7 @@ export class CodeCompletionAgent {
           id: ++c,
           filename: file,
           content: content,
-          identifier: c*34,
+          identifier: c-1,
         });
       }
     }
@@ -64,15 +65,33 @@ export class CodeCompletionAgent {
     });
   }
 
-  async searchIndex(query: string) {
+  async getContextFiles() {
     try {
-      const searchResult = this.indexer.search(query);
-      console.log('Search result', searchResult);
-      return searchResult[0];
+      const currentFile = await this.props.call('fileManager', 'getCurrentFile');
+      const content = await this.props.call('fileManager', 'readFile', currentFile);
+      const searchResult = this.indexer.search(content);
+      const fcps = await this.processResults(searchResult, currentFile);
+      const resolvedFcps = await Promise.all(fcps);
+      return resolvedFcps;
     } catch (error) {
-      console.log('Error searching index', error);
-      return null;
+      return [];
     }
   }
 
+  async processResults(results: any, currentFile: string) {
+    const rmResults = await results.filter(result => {
+      const document = this.Documents.find(doc => doc.id === Number(result.ref));
+      return document.filename !== currentFile;
+    });
+    const filteredResults = await rmResults.filter(result => result.score >= this.INDEX_THRESHOLD);
+    const topResults = filteredResults.slice(0, this.N_MATCHES);
+
+    const fileContentPairs = topResults.map(async result => {
+      const document = this.Documents.find(doc => doc.id === Number(result.ref));
+      const currentContent = await this.props.call('fileManager', 'readFile', document.filename);
+      return { file: document.filename, content: currentContent };
+    });
+    return fileContentPairs;
+  }
+
 }
diff --git a/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts b/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts
index c6ad381618..637f2b1422 100644
--- a/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts
+++ b/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts
@@ -12,7 +12,7 @@ export class RemoteInferencer implements ICompletions {
   max_history = 7
   model_op = RemoteBackendOPModel.CODELLAMA // default model operation change this to llama if necessary
   event: EventEmitter
-  test_env=false
+  test_env=true
   test_url="http://solcodertest.org"
 
   constructor(apiUrl?:string, completionUrl?:string) {
@@ -110,13 +110,16 @@ export class RemoteInferencer implements ICompletions {
     }
   }
 
-  async code_completion(prompt, promptAfter, options:IParams=CompletionParams): Promise<any> {
-    const payload = { prompt, 'context':promptAfter, "endpoint":"code_completion", ...options }
+  async code_completion(prompt, promptAfter, ctxFiles, fileName, options:IParams=CompletionParams): Promise<any> {
+    console.log("code_completion", ctxFiles)
+    const payload = { prompt, 'context':promptAfter, "endpoint":"code_completion",
+      'ctxFiles':ctxFiles, 'currentFileName':fileName, ...options }
     return this._makeRequest(payload, AIRequestType.COMPLETION)
   }
 
-  async code_insertion(msg_pfx, msg_sfx, options:IParams=InsertionParams): Promise<any> {
-    const payload = { "endpoint":"code_insertion", msg_pfx, msg_sfx, ...options, prompt: '' }
+  async code_insertion(msg_pfx, msg_sfx, ctxFiles, fileName, options:IParams=InsertionParams): Promise<any> {
+    const payload = { "endpoint":"code_insertion", msg_pfx, msg_sfx, 'ctxFiles':ctxFiles,
+      'currentFileName':fileName, ...options, prompt: '' }
     return this._makeRequest(payload, AIRequestType.COMPLETION)
   }
 
diff --git a/libs/remix-ai-core/src/types/types.ts b/libs/remix-ai-core/src/types/types.ts
index 1e5d9fb603..3e0417031f 100644
--- a/libs/remix-ai-core/src/types/types.ts
+++ b/libs/remix-ai-core/src/types/types.ts
@@ -50,8 +50,8 @@ export interface InferenceModel {
 }
 
 export interface ICompletions{
-  code_completion(context, params:IParams): Promise<any>;
-  code_insertion(msg_pfx, msg_sfx, params:IParams): Promise<any>;
+  code_completion(context, ctxFiles, fileName, params:IParams): Promise<any>;
+  code_insertion(msg_pfx, msg_sfx, ctxFiles, fileName, params:IParams): Promise<any>;
 }
 
 export interface IParams {

From 5c062d7f9b4754e6579854bb76decc77a287c595 Mon Sep 17 00:00:00 2001
From: STetsing <41009393+STetsing@users.noreply.github.com>
Date: Mon, 27 Jan 2025 14:08:36 +0100
Subject: [PATCH 03/16] added file extention context

---
 .../src/app/plugins/remixAIPlugin.tsx          |  2 --
 .../src/agents/completionAgent.ts              | 18 ++++++++++++------
 .../src/inferencers/remote/remoteInference.ts  |  1 -
 3 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/apps/remix-ide/src/app/plugins/remixAIPlugin.tsx b/apps/remix-ide/src/app/plugins/remixAIPlugin.tsx
index 69c415c1ed..df1d15f6bd 100644
--- a/apps/remix-ide/src/app/plugins/remixAIPlugin.tsx
+++ b/apps/remix-ide/src/app/plugins/remixAIPlugin.tsx
@@ -66,7 +66,6 @@ export class RemixAIPlugin extends ViewPlugin {
     }
     this.completionAgent = new CodeCompletionAgent(this)
 
-    // each 10 seconds update the workspace index
     setInterval(() => {
       console.log('Indexing workspace')
       this.completionAgent.indexWorkspace()
@@ -123,7 +122,6 @@ export class RemixAIPlugin extends ViewPlugin {
 
     const currentFile = await this.call('fileManager', 'getCurrentFile')
     const contextfiles = await this.completionAgent.getContextFiles()
-    console.log('completion Context files', contextfiles)
     if (this.isOnDesktop && !this.useRemoteInferencer) {
       return await this.call(this.remixDesktopPluginName, 'code_completion', prompt, promptAfter)
     } else {
diff --git a/libs/remix-ai-core/src/agents/completionAgent.ts b/libs/remix-ai-core/src/agents/completionAgent.ts
index dfc1a44493..11dcb903cd 100644
--- a/libs/remix-ai-core/src/agents/completionAgent.ts
+++ b/libs/remix-ai-core/src/agents/completionAgent.ts
@@ -25,6 +25,12 @@ export class CodeCompletionAgent {
     this.props = props;
     this.props.on('fileManager', 'fileAdded', (path) => { });
     this.props.on('filePanel', 'workspaceCreated', async () => { });
+    this.indexer =lunr(function () {
+      this.ref('id')
+      this.field('filename')
+      this.field('content')
+      this.field('Identifier');
+    });
   }
 
   async getDcocuments() {
@@ -45,7 +51,6 @@ export class CodeCompletionAgent {
         });
       }
     }
-    console.log('Documents', documents);
     return documents;
   }
 
@@ -69,7 +74,7 @@ export class CodeCompletionAgent {
     try {
       const currentFile = await this.props.call('fileManager', 'getCurrentFile');
       const content = await this.props.call('fileManager', 'readFile', currentFile);
-      const searchResult = this.indexer.search(content);
+      const searchResult = this.indexer.search(content)
       const fcps = await this.processResults(searchResult, currentFile);
       const resolvedFcps = await Promise.all(fcps);
       return resolvedFcps;
@@ -80,11 +85,12 @@ export class CodeCompletionAgent {
 
   async processResults(results: any, currentFile: string) {
     const rmResults = await results.filter(result => {
-      const document = this.Documents.find(doc => doc.id === Number(result.ref));
-      return document.filename !== currentFile;
+      return this.Documents.find(doc => doc.id === Number(result.ref)).filename !== currentFile;
+    });
+    const extResults = await rmResults.filter(result => {
+      return this.Documents.find(doc => doc.id === Number(result.ref)).filename.split('.').pop() === currentFile.split('.').pop();
     });
-    const filteredResults = await rmResults.filter(result => result.score >= this.INDEX_THRESHOLD);
-    const topResults = filteredResults.slice(0, this.N_MATCHES);
+    const topResults = await extResults.filter(result => result.score >= this.INDEX_THRESHOLD).slice(0, this.N_MATCHES);
 
     const fileContentPairs = topResults.map(async result => {
       const document = this.Documents.find(doc => doc.id === Number(result.ref));
diff --git a/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts b/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts
index 637f2b1422..c3914ea87d 100644
--- a/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts
+++ b/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts
@@ -111,7 +111,6 @@ export class RemoteInferencer implements ICompletions {
   }
 
   async code_completion(prompt, promptAfter, ctxFiles, fileName, options:IParams=CompletionParams): Promise<any> {
-    console.log("code_completion", ctxFiles)
     const payload = { prompt, 'context':promptAfter, "endpoint":"code_completion",
       'ctxFiles':ctxFiles, 'currentFileName':fileName, ...options }
     return this._makeRequest(payload, AIRequestType.COMPLETION)

From 5f4e10e0ef15dac423a57d83e8094918e5f8e4e0 Mon Sep 17 00:00:00 2001
From: STetsing <41009393+STetsing@users.noreply.github.com>
Date: Mon, 27 Jan 2025 14:09:06 +0100
Subject: [PATCH 04/16] minor

---
 libs/remix-ai-core/src/inferencers/remote/remoteInference.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts b/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts
index c3914ea87d..430c5ec348 100644
--- a/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts
+++ b/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts
@@ -12,7 +12,7 @@ export class RemoteInferencer implements ICompletions {
   max_history = 7
   model_op = RemoteBackendOPModel.CODELLAMA // default model operation change this to llama if necessary
   event: EventEmitter
-  test_env=true
+  test_env=false
   test_url="http://solcodertest.org"
 
   constructor(apiUrl?:string, completionUrl?:string) {

From 5302560a1336c9f230efa736a331e05ff9eb1aaa Mon Sep 17 00:00:00 2001
From: STetsing <41009393+STetsing@users.noreply.github.com>
Date: Mon, 3 Feb 2025 14:32:08 +0100
Subject: [PATCH 05/16] added max token support && event support for indexing

---
 .../src/app/plugins/remixAIPlugin.tsx         | 25 +++---
 .../src/agents/completionAgent.ts             | 79 +++++++++++++++++--
 .../src/inferencers/remote/remoteInference.ts |  2 +-
 3 files changed, 86 insertions(+), 20 deletions(-)

diff --git a/apps/remix-ide/src/app/plugins/remixAIPlugin.tsx b/apps/remix-ide/src/app/plugins/remixAIPlugin.tsx
index 14ba934f3b..312fab4f76 100644
--- a/apps/remix-ide/src/app/plugins/remixAIPlugin.tsx
+++ b/apps/remix-ide/src/app/plugins/remixAIPlugin.tsx
@@ -65,13 +65,6 @@ export class RemixAIPlugin extends ViewPlugin {
       this.initialize()
     }
     this.completionAgent = new CodeCompletionAgent(this)
-
-    setInterval(() => {
-      console.log('Indexing workspace')
-      this.completionAgent.indexWorkspace()
-    }, 60000)
-
-
     this.securityAgent = new SecurityAgent(this)
   }
 
@@ -119,11 +112,13 @@ export class RemixAIPlugin extends ViewPlugin {
     if (this.completionAgent.indexer == null || this.completionAgent.indexer == undefined) await this.completionAgent.indexWorkspace()
 
     const currentFile = await this.call('fileManager', 'getCurrentFile')
-    const contextfiles = await this.completionAgent.getContextFiles()
+    const contextfiles = await this.completionAgent.getContextFiles(prompt)
+    const refined = CodeCompletionAgent.refineContext(prompt, promptAfter)
+    console.log('contextfiles', contextfiles)
     if (this.isOnDesktop && !this.useRemoteInferencer) {
-      return await this.call(this.remixDesktopPluginName, 'code_completion', prompt, promptAfter)
+      return await this.call(this.remixDesktopPluginName, 'code_completion', refined[0], refined[1])
     } else {
-      return await this.remoteInferencer.code_completion(prompt, promptAfter, contextfiles, currentFile)
+      return await this.remoteInferencer.code_completion(refined[0], refined[1], contextfiles, currentFile)
     }
   }
 
@@ -185,11 +180,13 @@ export class RemixAIPlugin extends ViewPlugin {
     if (this.completionAgent.indexer == null || this.completionAgent.indexer == undefined) await this.completionAgent.indexWorkspace()
 
     const currentFile = await this.call('fileManager', 'getCurrentFile')
-    const contextfiles = this.completionAgent.getContextFiles()
+    const contextfiles = this.completionAgent.getContextFiles(msg_pfx)
+    const refined = CodeCompletionAgent.refineContext(msg_pfx, msg_sfx)
+
     if (this.isOnDesktop && !this.useRemoteInferencer) {
-      return await this.call(this.remixDesktopPluginName, 'code_insertion', msg_pfx, msg_sfx)
+      return await this.call(this.remixDesktopPluginName, 'code_insertion', refined[0], refined[1])
     } else {
-      return await this.remoteInferencer.code_insertion(msg_pfx, msg_sfx, contextfiles, currentFile)
+      return await this.remoteInferencer.code_insertion(refined[0], refined[1], contextfiles, currentFile)
     }
   }
 
@@ -212,7 +209,7 @@ export class RemixAIPlugin extends ViewPlugin {
     else {
       console.log("chatRequestBuffer is not empty. First process the last request.", this.chatRequestBuffer)
     }
-    _paq.push(['trackEvent', 'ai', 'remixAI_chat', 'askFromTerminal'])
+    _paq.push(['trackEvent', 'ai', 'remixAI', 'remixAI_chat'])
   }
 
   async ProcessChatRequestBuffer(params:IParams=GenerationParams){
diff --git a/libs/remix-ai-core/src/agents/completionAgent.ts b/libs/remix-ai-core/src/agents/completionAgent.ts
index 11dcb903cd..c31ff6a5be 100644
--- a/libs/remix-ai-core/src/agents/completionAgent.ts
+++ b/libs/remix-ai-core/src/agents/completionAgent.ts
@@ -7,6 +7,12 @@ interface Document {
   identifier: number;
 }
 
+interface indexT{
+  isIndexed: boolean;
+  lastIndexedTime?: number;
+  reason?: string;
+}
+
 enum SupportedFileExtensions {
   solidity = '.sol',
   vyper = '.vy',
@@ -20,17 +26,36 @@ export class CodeCompletionAgent {
   Documents: Document[] = [];
   INDEX_THRESHOLD = 0.1;
   N_MATCHES = 1;
+  indexed: indexT = {
+    isIndexed: false,
+    lastIndexedTime: 0,
+    reason: 'Init',
+  };
 
   constructor(props) {
     this.props = props;
-    this.props.on('fileManager', 'fileAdded', (path) => { });
-    this.props.on('filePanel', 'workspaceCreated', async () => { });
+    this.listenForChanges();
     this.indexer =lunr(function () {
       this.ref('id')
       this.field('filename')
       this.field('content')
       this.field('Identifier');
     });
+
+    // listen for file changes
+    this.props.on('fileManager', 'fileRemoved', (path) => { this.indexed.isIndexed = false; });
+
+    setInterval(() => {
+      this.indexWorkspace()
+    }, 60000)
+  }
+
+  listenForChanges() {
+    this.props.on('fileManager', 'fileAdded', (path) => { this.indexed = { isIndexed: false, reason:"fileAdded" } });
+    this.props.on('fileManager', 'fileRemoved', (path) => { this.indexed = { isIndexed: false, reason:"fileRemoved" } });
+    this.props.on('filePanel', 'workspaceCreated', () => { this.indexed = { isIndexed: false, reason:"workspaceCreated" } });
+    this.props.on('filePanel', 'workspaceRenamed', () => { this.indexed = { isIndexed: false, reason:"workspaceRenamed" }});
+    this.props.on('filePanel', 'workspaceDeleted', () => { this.indexed = { isIndexed: false, reason:"workspaceDeleted" } });
   }
 
   async getDcocuments() {
@@ -56,7 +81,6 @@ export class CodeCompletionAgent {
 
   indexWorkspace() {
     this.getDcocuments().then((documents) => {
-      this.Documents = documents;
       this.indexer =lunr(function () {
         this.ref('id')
         this.field('filename')
@@ -67,13 +91,19 @@ export class CodeCompletionAgent {
           this.add(doc);
         });
       });
+      this.Documents = documents;
     });
+
+    this.indexed = { isIndexed: true, lastIndexedTime: Date.now(), reason: 'init Indexing' };
   }
 
-  async getContextFiles() {
+  async getContextFiles(prompt) {
     try {
+      if (!this.indexed.isIndexed) {
+        this.indexWorkspace();
+      }
       const currentFile = await this.props.call('fileManager', 'getCurrentFile');
-      const content = await this.props.call('fileManager', 'readFile', currentFile);
+      const content = prompt;
       const searchResult = this.indexer.search(content)
       const fcps = await this.processResults(searchResult, currentFile);
       const resolvedFcps = await Promise.all(fcps);
@@ -100,4 +130,43 @@ export class CodeCompletionAgent {
     return fileContentPairs;
   }
 
+  // TODO rm context files length
+  static refineContext(words1: string, words2: string, maxWords: number=6500) {
+    // Avoid model out of context
+    const totalWords = words1.length + words2.length;
+    if (totalWords <= maxWords) {
+      console.log('total words less than max words', totalWords);
+      return [words1, words2];
+    }
+    const halfMaxWords = Math.floor(maxWords / 2);
+
+    let takeFromText1 = words1.length < halfMaxWords ? words1.length : halfMaxWords;
+    let takeFromText2 = words2.length < halfMaxWords ? words2.length : halfMaxWords;
+
+    // Adjust if one text is taking more than half, we balance by limiting the other text
+    if (words1.length < halfMaxWords && words2.length + words1.length <= maxWords) {
+      takeFromText2 = Math.min(words2.length, maxWords - words1.length);
+    } else if (words2.length < halfMaxWords && words1.length + words2.length <= maxWords) {
+      takeFromText1 = Math.min(words1.length, maxWords - words2.length);
+    } else if (words1.length < halfMaxWords && words2.length + words1.length >= maxWords) {
+      takeFromText2 = Math.min(words2.length, maxWords - words1.length);
+    } else if (words2.length > halfMaxWords && words1.length + words2.length <= maxWords) {
+      takeFromText1 = Math.min(words1.length, maxWords - words2.length);
+    }
+
+    const splicedText1 = words1.slice(words1.length - takeFromText1);
+    const splicedText2 = words2.slice(words2.length - takeFromText2);
+    console.log('take from text 1', takeFromText1)
+    console.log('take from text 2', takeFromText2)
+
+    console.log('Spliced text 1 length:', splicedText1.length);
+    console.log('Spliced text 2 length:', splicedText2.length);
+    console.log('initial word 1 length:', words1.length);
+    console.log('initial word 2 length:', words2.length);
+    console.log('text 1:', splicedText1);
+    console.log('text 2:', splicedText2);
+
+    return [splicedText1 , splicedText2]
+  }
+
 }
diff --git a/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts b/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts
index a8fa702fc7..92eb0aea6a 100644
--- a/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts
+++ b/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts
@@ -12,7 +12,7 @@ export class RemoteInferencer implements ICompletions {
   max_history = 7
   model_op = RemoteBackendOPModel.CODELLAMA // default model operation change this to llama if necessary
   event: EventEmitter
-  test_env=false
+  test_env=true
   test_url="http://solcodertest.org"
 
   constructor(apiUrl?:string, completionUrl?:string) {

From d68b0a5f3d82087bb4a39cb82c76ed4d8e2f9b2d Mon Sep 17 00:00:00 2001
From: STetsing <41009393+STetsing@users.noreply.github.com>
Date: Mon, 3 Feb 2025 14:32:56 +0100
Subject: [PATCH 06/16] minor

---
 libs/remix-ai-core/src/agents/completionAgent.ts | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/libs/remix-ai-core/src/agents/completionAgent.ts b/libs/remix-ai-core/src/agents/completionAgent.ts
index c31ff6a5be..e332c1f938 100644
--- a/libs/remix-ai-core/src/agents/completionAgent.ts
+++ b/libs/remix-ai-core/src/agents/completionAgent.ts
@@ -135,7 +135,6 @@ export class CodeCompletionAgent {
     // Avoid model out of context
     const totalWords = words1.length + words2.length;
     if (totalWords <= maxWords) {
-      console.log('total words less than max words', totalWords);
       return [words1, words2];
     }
     const halfMaxWords = Math.floor(maxWords / 2);
@@ -156,16 +155,6 @@ export class CodeCompletionAgent {
 
     const splicedText1 = words1.slice(words1.length - takeFromText1);
     const splicedText2 = words2.slice(words2.length - takeFromText2);
-    console.log('take from text 1', takeFromText1)
-    console.log('take from text 2', takeFromText2)
-
-    console.log('Spliced text 1 length:', splicedText1.length);
-    console.log('Spliced text 2 length:', splicedText2.length);
-    console.log('initial word 1 length:', words1.length);
-    console.log('initial word 2 length:', words2.length);
-    console.log('text 1:', splicedText1);
-    console.log('text 2:', splicedText2);
-
     return [splicedText1 , splicedText2]
   }
 

From 9c03ba221b6094e996492e9a9d6dac38a5b33ce7 Mon Sep 17 00:00:00 2001
From: STetsing <41009393+STetsing@users.noreply.github.com>
Date: Wed, 5 Feb 2025 14:19:23 +0100
Subject: [PATCH 07/16] moved refinement to backend

---
 .../src/app/plugins/remixAIPlugin.tsx         | 13 ++++-----
 .../src/agents/completionAgent.ts             | 28 -------------------
 2 files changed, 5 insertions(+), 36 deletions(-)

diff --git a/apps/remix-ide/src/app/plugins/remixAIPlugin.tsx b/apps/remix-ide/src/app/plugins/remixAIPlugin.tsx
index 312fab4f76..5853a1e6f9 100644
--- a/apps/remix-ide/src/app/plugins/remixAIPlugin.tsx
+++ b/apps/remix-ide/src/app/plugins/remixAIPlugin.tsx
@@ -113,12 +113,10 @@ export class RemixAIPlugin extends ViewPlugin {
 
     const currentFile = await this.call('fileManager', 'getCurrentFile')
     const contextfiles = await this.completionAgent.getContextFiles(prompt)
-    const refined = CodeCompletionAgent.refineContext(prompt, promptAfter)
-    console.log('contextfiles', contextfiles)
     if (this.isOnDesktop && !this.useRemoteInferencer) {
-      return await this.call(this.remixDesktopPluginName, 'code_completion', refined[0], refined[1])
+      return await this.call(this.remixDesktopPluginName, 'code_completion', prompt, promptAfter)
     } else {
-      return await this.remoteInferencer.code_completion(refined[0], refined[1], contextfiles, currentFile)
+      return await this.remoteInferencer.code_completion(prompt, promptAfter, contextfiles, currentFile)
     }
   }
 
@@ -180,13 +178,12 @@ export class RemixAIPlugin extends ViewPlugin {
     if (this.completionAgent.indexer == null || this.completionAgent.indexer == undefined) await this.completionAgent.indexWorkspace()
 
     const currentFile = await this.call('fileManager', 'getCurrentFile')
-    const contextfiles = this.completionAgent.getContextFiles(msg_pfx)
-    const refined = CodeCompletionAgent.refineContext(msg_pfx, msg_sfx)
+    const contextfiles = await this.completionAgent.getContextFiles(msg_pfx)
 
     if (this.isOnDesktop && !this.useRemoteInferencer) {
-      return await this.call(this.remixDesktopPluginName, 'code_insertion', refined[0], refined[1])
+      return await this.call(this.remixDesktopPluginName, 'code_insertion', msg_pfx, msg_sfx)
     } else {
-      return await this.remoteInferencer.code_insertion(refined[0], refined[1], contextfiles, currentFile)
+      return await this.remoteInferencer.code_insertion( msg_pfx, msg_sfx, contextfiles, currentFile)
     }
   }
 
diff --git a/libs/remix-ai-core/src/agents/completionAgent.ts b/libs/remix-ai-core/src/agents/completionAgent.ts
index e332c1f938..b76a61ee4f 100644
--- a/libs/remix-ai-core/src/agents/completionAgent.ts
+++ b/libs/remix-ai-core/src/agents/completionAgent.ts
@@ -130,32 +130,4 @@ export class CodeCompletionAgent {
     return fileContentPairs;
   }
 
-  // TODO rm context files length
-  static refineContext(words1: string, words2: string, maxWords: number=6500) {
-    // Avoid model out of context
-    const totalWords = words1.length + words2.length;
-    if (totalWords <= maxWords) {
-      return [words1, words2];
-    }
-    const halfMaxWords = Math.floor(maxWords / 2);
-
-    let takeFromText1 = words1.length < halfMaxWords ? words1.length : halfMaxWords;
-    let takeFromText2 = words2.length < halfMaxWords ? words2.length : halfMaxWords;
-
-    // Adjust if one text is taking more than half, we balance by limiting the other text
-    if (words1.length < halfMaxWords && words2.length + words1.length <= maxWords) {
-      takeFromText2 = Math.min(words2.length, maxWords - words1.length);
-    } else if (words2.length < halfMaxWords && words1.length + words2.length <= maxWords) {
-      takeFromText1 = Math.min(words1.length, maxWords - words2.length);
-    } else if (words1.length < halfMaxWords && words2.length + words1.length >= maxWords) {
-      takeFromText2 = Math.min(words2.length, maxWords - words1.length);
-    } else if (words2.length > halfMaxWords && words1.length + words2.length <= maxWords) {
-      takeFromText1 = Math.min(words1.length, maxWords - words2.length);
-    }
-
-    const splicedText1 = words1.slice(words1.length - takeFromText1);
-    const splicedText2 = words2.slice(words2.length - takeFromText2);
-    return [splicedText1 , splicedText2]
-  }
-
 }

From b11c8d2016a7504bd41c0e1814341901139d7b22 Mon Sep 17 00:00:00 2001
From: STetsing <41009393+STetsing@users.noreply.github.com>
Date: Thu, 6 Feb 2025 11:23:10 +0100
Subject: [PATCH 08/16] minor

---
 apps/vyper/src/app/utils/remix-client.tsx | 1 -
 1 file changed, 1 deletion(-)

diff --git a/apps/vyper/src/app/utils/remix-client.tsx b/apps/vyper/src/app/utils/remix-client.tsx
index fb09e559c8..a52e34ab8a 100644
--- a/apps/vyper/src/app/utils/remix-client.tsx
+++ b/apps/vyper/src/app/utils/remix-client.tsx
@@ -69,7 +69,6 @@ export class RemixClient extends PluginClient<any, CustomRemixApi> {
     try {
       // TODO: remove! no formatting required since already handled on server
       const file = await this.client.call('fileManager', 'getCurrentFile')
-      
       const content = await this.client.call('fileManager', 'readFile', file)
       const messageAI = `Vyper code: ${content}\n error message: ${message}\n explain why the error occurred and how to fix it.`
 

From ce7aa9e91b8b721d5efd07efd4d5bb69aa69516e Mon Sep 17 00:00:00 2001
From: STetsing <41009393+STetsing@users.noreply.github.com>
Date: Thu, 6 Feb 2025 12:24:18 +0100
Subject: [PATCH 09/16] minor

---
 libs/remix-ai-core/src/agents/completionAgent.ts             | 1 -
 libs/remix-ai-core/src/inferencers/remote/remoteInference.ts | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/libs/remix-ai-core/src/agents/completionAgent.ts b/libs/remix-ai-core/src/agents/completionAgent.ts
index b76a61ee4f..04b1ab7e79 100644
--- a/libs/remix-ai-core/src/agents/completionAgent.ts
+++ b/libs/remix-ai-core/src/agents/completionAgent.ts
@@ -20,7 +20,6 @@ enum SupportedFileExtensions {
 }
 
 export class CodeCompletionAgent {
-  private workspacesIndexes = new Map<string, any>();
   props: any;
   indexer: any;
   Documents: Document[] = [];
diff --git a/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts b/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts
index 92eb0aea6a..a8fa702fc7 100644
--- a/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts
+++ b/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts
@@ -12,7 +12,7 @@ export class RemoteInferencer implements ICompletions {
   max_history = 7
   model_op = RemoteBackendOPModel.CODELLAMA // default model operation change this to llama if necessary
   event: EventEmitter
-  test_env=true
+  test_env=false
   test_url="http://solcodertest.org"
 
   constructor(apiUrl?:string, completionUrl?:string) {

From 8108c7c6e689867169d20720100b02a04cc03d01 Mon Sep 17 00:00:00 2001
From: STetsing <41009393+STetsing@users.noreply.github.com>
Date: Thu, 6 Feb 2025 15:21:34 +0100
Subject: [PATCH 10/16] Update package.json rm flexsearch

---
 package.json | 1 -
 1 file changed, 1 deletion(-)

diff --git a/package.json b/package.json
index 0f682f3d24..385a5d9a50 100644
--- a/package.json
+++ b/package.json
@@ -153,7 +153,6 @@
     "express-ws": "^5.0.2",
     "file-path-filter": "^3.0.2",
     "file-saver": "^2.0.5",
-    "flexsearch": "^0.7.43",
     "form-data": "^4.0.0",
     "formik": "^2.4.5",
     "from-exponential": "1.1.1",

From ed76d02d4546188bc31c08685a670f1ffac31220 Mon Sep 17 00:00:00 2001
From: STetsing <41009393+STetsing@users.noreply.github.com>
Date: Thu, 6 Feb 2025 15:24:21 +0100
Subject: [PATCH 11/16] rm dynamic buttons

---
 .../src/lib/components/dynamicButtons.tsx     | 22 -------------------
 1 file changed, 22 deletions(-)
 delete mode 100644 libs/remix-ui/remix-ai/src/lib/components/dynamicButtons.tsx

diff --git a/libs/remix-ui/remix-ai/src/lib/components/dynamicButtons.tsx b/libs/remix-ui/remix-ai/src/lib/components/dynamicButtons.tsx
deleted file mode 100644
index 181e4d3247..0000000000
--- a/libs/remix-ui/remix-ai/src/lib/components/dynamicButtons.tsx
+++ /dev/null
@@ -1,22 +0,0 @@
-import React from 'react';
-
-interface ButtonProps {
-  label: string;
-  onClick: () => void;
-  icon?: string;
-}
-
-const DynamicButtons: React.FC<{ buttons: ButtonProps[] }> = ({ buttons }) => {
-  return (
-    <div className="dynamic-buttons">
-      {buttons.map((button, index) => (
-        <button key={index} onClick={button.onClick}>
-          {button.icon && <i className={button.icon}></i>}
-          {button.label}
-        </button>
-      ))}
-    </div>
-  );
-};
-
-export default DynamicButtons;
\ No newline at end of file

From e3a3dff38d141f1335f3af111c66626b3c60fdbc Mon Sep 17 00:00:00 2001
From: STetsing <41009393+STetsing@users.noreply.github.com>
Date: Thu, 6 Feb 2025 15:30:37 +0100
Subject: [PATCH 12/16] added some comments

---
 libs/remix-ai-core/src/agents/completionAgent.ts | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/libs/remix-ai-core/src/agents/completionAgent.ts b/libs/remix-ai-core/src/agents/completionAgent.ts
index 04b1ab7e79..bc9a499eff 100644
--- a/libs/remix-ai-core/src/agents/completionAgent.ts
+++ b/libs/remix-ai-core/src/agents/completionAgent.ts
@@ -41,9 +41,6 @@ export class CodeCompletionAgent {
       this.field('Identifier');
     });
 
-    // listen for file changes
-    this.props.on('fileManager', 'fileRemoved', (path) => { this.indexed.isIndexed = false; });
-
     setInterval(() => {
       this.indexWorkspace()
     }, 60000)
@@ -99,7 +96,7 @@ export class CodeCompletionAgent {
   async getContextFiles(prompt) {
     try {
       if (!this.indexed.isIndexed) {
-        this.indexWorkspace();
+        await this.indexWorkspace();
       }
       const currentFile = await this.props.call('fileManager', 'getCurrentFile');
       const content = prompt;
@@ -113,14 +110,22 @@ export class CodeCompletionAgent {
   }
 
   async processResults(results: any, currentFile: string) {
+
+    // remove the current file name from the results list
     const rmResults = await results.filter(result => {
       return this.Documents.find(doc => doc.id === Number(result.ref)).filename !== currentFile;
     });
+
+    // filter out the results which have the same extension as the current file.
+    // Do not mix and match file extensions as this will lead to incorrect completions
     const extResults = await rmResults.filter(result => {
       return this.Documents.find(doc => doc.id === Number(result.ref)).filename.split('.').pop() === currentFile.split('.').pop();
     });
+
+    // filter out the results which have a score less than the INDEX_THRESHOLD
     const topResults = await extResults.filter(result => result.score >= this.INDEX_THRESHOLD).slice(0, this.N_MATCHES);
 
+    // get the LATEST content of the top results in case the file has been modified and not indexed yet
     const fileContentPairs = topResults.map(async result => {
       const document = this.Documents.find(doc => doc.id === Number(result.ref));
       const currentContent = await this.props.call('fileManager', 'readFile', document.filename);

From 3232af0877ad3f249395e895c1b3c62007bd4ba3 Mon Sep 17 00:00:00 2001
From: STetsing <41009393+STetsing@users.noreply.github.com>
Date: Wed, 12 Feb 2025 10:16:40 +0100
Subject: [PATCH 13/16] moved to copyfoldertojson

---
 .../src/agents/completionAgent.ts             | 24 +++++++++----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/libs/remix-ai-core/src/agents/completionAgent.ts b/libs/remix-ai-core/src/agents/completionAgent.ts
index bc9a499eff..d0df5b1bcf 100644
--- a/libs/remix-ai-core/src/agents/completionAgent.ts
+++ b/libs/remix-ai-core/src/agents/completionAgent.ts
@@ -55,24 +55,23 @@ export class CodeCompletionAgent {
   }
 
   async getDcocuments() {
-    const documents: Document[] = [];
-    const dirs = await this.props.call('fileManager', 'dirList', '/');
-    let c = 0;
-    for (const dir of dirs) {
-      const files = await this.props.call('fileManager', 'fileList', dir);
-      for (const file of files) {
-        const content = await this.props.call('fileManager', 'readFile', file);
-        // filter out any  SupportedFileExtensions
-        if (! Object.values(SupportedFileExtensions).some(ext => file.endsWith(ext))) continue;
+    try {
+      const documents: Document[] = [];
+      const jsonDirsContracts = await this.props.call('fileManager', 'copyFolderToJson', '/').then((res) => res.contracts);
+      let c = 0;
+      for (const file in jsonDirsContracts.children) {
+        if (!Object.values(SupportedFileExtensions).some(ext => file.endsWith(ext))) continue;
         documents.push({
           id: ++c,
           filename: file,
-          content: content,
-          identifier: c-1,
+          content: jsonDirsContracts.children[file].content,
+          identifier: c - 1,
         });
       }
+      return documents;
+    } catch (error) {
+      return [];
     }
-    return documents;
   }
 
   indexWorkspace() {
@@ -98,6 +97,7 @@ export class CodeCompletionAgent {
       if (!this.indexed.isIndexed) {
         await this.indexWorkspace();
       }
+
       const currentFile = await this.props.call('fileManager', 'getCurrentFile');
       const content = prompt;
       const searchResult = this.indexer.search(content)

From 8f4ad2398be5de3aaad55b12fba7857bce244fc8 Mon Sep 17 00:00:00 2001
From: STetsing <41009393+STetsing@users.noreply.github.com>
Date: Wed, 12 Feb 2025 14:48:53 +0100
Subject: [PATCH 14/16] added local import files to context

---
 .../src/agents/completionAgent.ts             | 32 +++++++++++++++++++
 .../src/inferencers/remote/remoteInference.ts |  2 +-
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/libs/remix-ai-core/src/agents/completionAgent.ts b/libs/remix-ai-core/src/agents/completionAgent.ts
index d0df5b1bcf..ae2866f91c 100644
--- a/libs/remix-ai-core/src/agents/completionAgent.ts
+++ b/libs/remix-ai-core/src/agents/completionAgent.ts
@@ -74,6 +74,30 @@ export class CodeCompletionAgent {
     }
   }
 
+  async getLocalImports(fileContent: string, currentFile: string) {
+    try {
+      const lines = fileContent.split('\n');
+      const imports = [];
+
+      for (const line of lines) {
+        const trimmedLine = line.trim();
+        if (trimmedLine.startsWith('import')) {
+          const parts = trimmedLine.split(' ');
+          if (parts.length >= 2) {
+            const importPath = parts[1].replace(/['";]/g, '');
+            imports.push(importPath);
+          }
+        }
+      }
+      // Only local imports are those files that are in the workspace
+      const localImports = this.Documents.length >0 ? imports.filter((imp) => {return this.Documents.find((doc) => doc.filename === imp);}) : [];
+
+      return localImports;
+    } catch (error) {
+      return [];
+    }
+  }
+
   indexWorkspace() {
     this.getDcocuments().then((documents) => {
       this.indexer =lunr(function () {
@@ -131,6 +155,14 @@ export class CodeCompletionAgent {
       const currentContent = await this.props.call('fileManager', 'readFile', document.filename);
       return { file: document.filename, content: currentContent };
     });
+
+    const localImports = await this.getLocalImports(await this.props.call('fileManager', 'readFile', currentFile), currentFile);
+    // check if the local import is in fileContentPairs file
+    for (const li of localImports) {
+      if (fileContentPairs.find(fcp => fcp.file === li)) continue;
+      const currentContent = await this.props.call('fileManager', 'readFile', li);
+      fileContentPairs.push({ file: li, content: currentContent });
+    }
     return fileContentPairs;
   }
 
diff --git a/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts b/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts
index a8fa702fc7..92eb0aea6a 100644
--- a/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts
+++ b/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts
@@ -12,7 +12,7 @@ export class RemoteInferencer implements ICompletions {
   max_history = 7
   model_op = RemoteBackendOPModel.CODELLAMA // default model operation change this to llama if necessary
   event: EventEmitter
-  test_env=false
+  test_env=true
   test_url="http://solcodertest.org"
 
   constructor(apiUrl?:string, completionUrl?:string) {

From 5e02eb77cc01d49012459305520d16c20fcc5ad7 Mon Sep 17 00:00:00 2001
From: STetsing <41009393+STetsing@users.noreply.github.com>
Date: Wed, 12 Feb 2025 17:12:54 +0100
Subject: [PATCH 15/16] minot

---
 libs/remix-ai-core/src/inferencers/remote/remoteInference.ts    | 2 +-
 .../editor/src/lib/providers/inlineCompletionProvider.ts        | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts b/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts
index 92eb0aea6a..a8fa702fc7 100644
--- a/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts
+++ b/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts
@@ -12,7 +12,7 @@ export class RemoteInferencer implements ICompletions {
   max_history = 7
   model_op = RemoteBackendOPModel.CODELLAMA // default model operation change this to llama if necessary
   event: EventEmitter
-  test_env=true
+  test_env=false
   test_url="http://solcodertest.org"
 
   constructor(apiUrl?:string, completionUrl?:string) {
diff --git a/libs/remix-ui/editor/src/lib/providers/inlineCompletionProvider.ts b/libs/remix-ui/editor/src/lib/providers/inlineCompletionProvider.ts
index 5f15022b10..0525275704 100644
--- a/libs/remix-ui/editor/src/lib/providers/inlineCompletionProvider.ts
+++ b/libs/remix-ui/editor/src/lib/providers/inlineCompletionProvider.ts
@@ -106,7 +106,6 @@ export class RemixInLineCompletionProvider implements monacoTypes.languages.Inli
         const generatedText = output // no need to clean it. should already be
 
         this.task = 'code_insertion'
-        _paq.push(['trackEvent', 'ai', 'remixAI', this.task])
         const item: monacoTypes.languages.InlineCompletion = {
           insertText: generatedText,
           range: new monaco.Range(position.lineNumber, position.column, position.lineNumber, position.column)

From b6347ce13e3d4a317be0b526ff38a8d4fe4b4629 Mon Sep 17 00:00:00 2001
From: STetsing <41009393+STetsing@users.noreply.github.com>
Date: Wed, 12 Feb 2025 17:25:47 +0100
Subject: [PATCH 16/16] minor

---
 .../editor/src/lib/providers/inlineCompletionProvider.ts         | 1 -
 1 file changed, 1 deletion(-)

diff --git a/libs/remix-ui/editor/src/lib/providers/inlineCompletionProvider.ts b/libs/remix-ui/editor/src/lib/providers/inlineCompletionProvider.ts
index 0525275704..8c9443cdbe 100644
--- a/libs/remix-ui/editor/src/lib/providers/inlineCompletionProvider.ts
+++ b/libs/remix-ui/editor/src/lib/providers/inlineCompletionProvider.ts
@@ -162,7 +162,6 @@ export class RemixInLineCompletionProvider implements monacoTypes.languages.Inli
     let clean = data
     // if clean starts with a comment, remove it
     if (clean.startsWith('//') || clean.startsWith('/*') || clean.startsWith('*') || clean.startsWith('*/')){
-      console.log("clean starts with comment")
       return ""
     }