From 52f594e04da03c566f65235ef5c4fe790c253179 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Tetsing?= Date: Tue, 23 Jul 2024 16:10:11 +0200 Subject: [PATCH] streaming inference result on desktop --- .../src/app/components/container.tsx | 6 +- .../src/app/plugins/remixAIPlugin.tsx | 46 ++- apps/remix-ide/src/remixEngine.js | 2 +- .../src/lib/InferenceServerManager.ts | 263 +++++++++++++++--- .../remixdesktop/src/plugins/remixAIDektop.ts | 53 +++- apps/vyper/src/app/utils/remix-client.tsx | 2 +- libs/remix-ai-core/src/index.ts | 11 +- .../src/inferencers/remote/remoteInference.ts | 62 ++++- libs/remix-ai-core/src/types/models.ts | 38 ++- libs/remix-ai-core/src/types/types.ts | 10 + .../editor/src/lib/remix-ui-editor.tsx | 12 +- .../remix-ai/src/lib/components/Default.tsx | 26 +- libs/remix-ui/renderer/src/lib/renderer.tsx | 4 +- .../settings/src/lib/remix-ui-settings.tsx | 2 +- libs/remix-ui/tabs/src/lib/remix-ui-tabs.tsx | 6 +- .../terminal/src/lib/remix-ui-terminal.tsx | 8 +- 16 files changed, 432 insertions(+), 119 deletions(-) diff --git a/apps/circuit-compiler/src/app/components/container.tsx b/apps/circuit-compiler/src/app/components/container.tsx index 40cf20f5df..f50d148e62 100644 --- a/apps/circuit-compiler/src/app/components/container.tsx +++ b/apps/circuit-compiler/src/app/components/container.tsx @@ -73,7 +73,7 @@ export function Container () { explain why the error occurred and how to fix it. ` // @ts-ignore - await circuitApp.plugin.call('solcoder', 'error_explaining', message) + await circuitApp.plugin.call('remixAI', 'error_explaining', message) } else { const message = ` error message: ${error} @@ -81,7 +81,7 @@ export function Container () { explain why the error occurred and how to fix it. ` // @ts-ignore - await circuitApp.plugin.call('solcoder', 'error_explaining', message) + await circuitApp.plugin.call('remixAI', 'error_explaining', message) } } else { const error = report.message @@ -91,7 +91,7 @@ export function Container () { explain why the error occurred and how to fix it. ` // @ts-ignore - await circuitApp.plugin.call('solcoder', 'error_explaining', message) + await circuitApp.plugin.call('remixAI', 'error_explaining', message) } } diff --git a/apps/remix-ide/src/app/plugins/remixAIPlugin.tsx b/apps/remix-ide/src/app/plugins/remixAIPlugin.tsx index 0b0d27b614..7a6687c3c4 100644 --- a/apps/remix-ide/src/app/plugins/remixAIPlugin.tsx +++ b/apps/remix-ide/src/app/plugins/remixAIPlugin.tsx @@ -3,7 +3,8 @@ import { ViewPlugin } from '@remixproject/engine-web' import { Plugin } from '@remixproject/engine'; import { RemixAITab } from '@remix-ui/remix-ai' import React from 'react'; -import { ICompletions, IModel, RemoteInferencer } from '@remix/remix-ai-core'; +import { ICompletions, IModel, RemoteInferencer, IRemoteModel } from '@remix/remix-ai-core'; +import { resourceUsage } from 'process'; const profile = { name: 'remixAI', @@ -11,7 +12,7 @@ const profile = { methods: ['code_generation', 'code_completion', "solidity_answer", "code_explaining", "code_insertion", "error_explaining", - "initializeRemixAI"], + "initialize"], events: [], icon: 'assets/img/remix-logo-blue.png', description: 'RemixAI provides AI services to Remix IDE.', @@ -25,7 +26,6 @@ const profile = { export class RemixAIPlugin extends ViewPlugin { isOnDesktop:boolean = false aiIsActivated:boolean = false - selectedModel:IModel = null readonly remixDesktopPluginName = 'remixAID' remoteInferencer:RemoteInferencer = null @@ -34,7 +34,7 @@ export class RemixAIPlugin extends ViewPlugin { super(profile) this.isOnDesktop = inDesktop - // not user machine ressource for remote inferencing + // user machine dont use ressource for remote inferencing } onActivation(): void { @@ -45,15 +45,18 @@ export class RemixAIPlugin extends ViewPlugin { } } - async initializeRemixAI(model: IModel) { - this.selectedModel = model + async initialize(model1?:IModel, model2?:IModel, remoteModel?:IRemoteModel){ if (this.isOnDesktop) { - this.call(this.remixDesktopPluginName, 'initializeModelBackend', this.selectedModel) + this.call(this.remixDesktopPluginName, 'initializeModelBackend', false, model1, model2) + this.on(this.remixDesktopPluginName, 'onStreamResult', (value) => { + console.log('onStreamResult remixai plugin', value) + this.call('terminal', 'log', { type: 'log', value: value }) + }) } else { // on browser console.log('Initializing RemixAIPlugin on browser') - this.remoteInferencer = new RemoteInferencer(this) + this.remoteInferencer = new RemoteInferencer(remoteModel?.apiUrl, remoteModel?.completionUrl) } this.aiIsActivated = true @@ -61,6 +64,7 @@ export class RemixAIPlugin extends ViewPlugin { } async code_generation(prompt: string): Promise { + console.log('code_generation') if (this.isOnDesktop) { return this.call(this.remixDesktopPluginName, 'code_generation', prompt) } else { @@ -77,27 +81,41 @@ export class RemixAIPlugin extends ViewPlugin { } async solidity_answer(prompt: string): Promise { + this.call('terminal', 'log', { type: 'aitypewriterwarning', value: `\n\nWaiting for RemixAI answer...` }) + + let result if (this.isOnDesktop) { - return this.call(this.remixDesktopPluginName, 'solidity_answer', prompt) + result = this.call(this.remixDesktopPluginName, 'solidity_answer', prompt) } else { - return this.remoteInferencer.solidity_answer(prompt) + result = this.remoteInferencer.solidity_answer(prompt) } + this.call('terminal', 'log', { type: 'aitypewriterwarning', value: result }) + } async code_explaining(prompt: string): Promise { + this.call('terminal', 'log', { type: 'aitypewriterwarning', value: `\n\nWaiting for RemixAI answer...` }) + + let result if (this.isOnDesktop) { - return this.call(this.remixDesktopPluginName, 'code_explaining', prompt) + result = await this.call(this.remixDesktopPluginName, 'code_explaining', prompt) + } else { - return this.remoteInferencer.code_explaining(prompt) + result = this.remoteInferencer.code_explaining(prompt) } + if (result) this.call('terminal', 'log', { type: 'aitypewriterwarning', value: result }) } async error_explaining(prompt: string): Promise { + this.call('terminal', 'log', { type: 'aitypewriterwarning', value: `\n\nWaiting for RemixAI answer...` }) + + let result if (this.isOnDesktop) { - return this.call(this.remixDesktopPluginName, 'error_explaining', prompt) + result = await this.call(this.remixDesktopPluginName, 'error_explaining', prompt) } else { - return this.remoteInferencer.error_explaining(prompt) + result = await this.remoteInferencer.error_explaining(prompt) } + this.call('terminal', 'log', { type: 'aitypewriterwarning', value: result }) } async code_insertion(msg_pfx: string, msg_sfx: string): Promise { diff --git a/apps/remix-ide/src/remixEngine.js b/apps/remix-ide/src/remixEngine.js index 00c0a40b83..d071fb464e 100644 --- a/apps/remix-ide/src/remixEngine.js +++ b/apps/remix-ide/src/remixEngine.js @@ -26,8 +26,8 @@ export class RemixEngine extends Engine { if (name === 'compilerloader') return { queueTimeout: 60000 * 4 } if (name === 'filePanel') return { queueTimeout: 60000 * 20 } if (name === 'fileManager') return { queueTimeout: 60000 * 20 } - if (name === 'solcoder') return { queueTimeout: 60000 * 2 } if (name === 'remixAID') return { queueTimeout: 60000 * 20 } + if (name === 'remixAI') return { queueTimeout: 60000 * 20 } if (name === 'cookbookdev') return { queueTimeout: 60000 * 3 } return { queueTimeout: 10000 } } diff --git a/apps/remixdesktop/src/lib/InferenceServerManager.ts b/apps/remixdesktop/src/lib/InferenceServerManager.ts index f216339fe5..69a2cfed59 100644 --- a/apps/remixdesktop/src/lib/InferenceServerManager.ts +++ b/apps/remixdesktop/src/lib/InferenceServerManager.ts @@ -1,77 +1,150 @@ -import path from 'path'; +import path, { resolve } from 'path'; const { spawn } = require('child_process'); // eslint-disable-line import fs from 'fs'; import axios from "axios"; import { EventEmitter } from 'events'; -import { ICompletions, IModel, IParams } from "@remix/remix-ai-core"; -import { getInsertionPrompt } from "@remix/remix-ai-core"; - -const completionParams:IParams = { - temperature: 0.8, - topK: 40, - topP: 0.92, - max_new_tokens: 15, -} +import { ICompletions, IModel, IParams, InsertionParams, + CompletionParams, GenerationParams, ModelType, + IStreamResponse } from "../../../../libs/remix-ai-core/src/index" + +class ServerStatusTimer { + private intervalId: NodeJS.Timeout | null = null; + public interval: number; + private task: () => void; + + constructor(task: () => void, interval: number) { + this.task = task; + this.interval = interval; + } + + start(): void { + if (this.intervalId === null) { + this.intervalId = setInterval(() => { + this.task(); + }, this.interval); + } + } -const insertionParams:IParams = { - temperature: 0.8, - topK: 40, - topP: 0.92, - max_new_tokens: 150, + stop(): void { + if (this.intervalId !== null) { + clearInterval(this.intervalId); + this.intervalId = null; + } + } + + isRunning(): boolean { + return this.intervalId !== null; + } } export class InferenceManager implements ICompletions { isReady: boolean = false - selectedModel: any - modelPath: string + selectedModels: IModel[] = [] event: EventEmitter modelCacheDir: string = undefined isInferencing: boolean = false - inferenceProcess: any=null - inferenceURL = 'http://127.0.0.1:5501' - static instance=null + private inferenceProcess: any=null + port = 5501 + inferenceURL = 'http://127.0.0.1:' + this.port + private static instance=null + stateTimer: ServerStatusTimer - private constructor(model:IModel, modelDir:string) { - this.selectedModel = model + private constructor(modelDir:string) { this.event = new EventEmitter() this.modelCacheDir = path.join(modelDir, 'models') + this.stateTimer= new ServerStatusTimer(() => { this._processStatus()}, 20000) } - static getInstance(model:IModel, modelDir:string){ + static getInstance(modelDir:string){ if (!InferenceManager.instance) { // check if ther is a process already running - if (!model || !modelDir) { - console.error('Model and model directory is required to create InferenceManager instance') + if (!modelDir) { + console.error('model directory is required to create InferenceManager instance') return null } console.log('Creating new InferenceManager instance') - InferenceManager.instance = new InferenceManager(model, modelDir) + InferenceManager.instance = new InferenceManager(modelDir) } return InferenceManager.instance } - async init() { + // init the backend with a new model + async init(model:IModel) { try { - await this._downloadModel(this.selectedModel) + await this._downloadModel(model) - if (this.modelPath === undefined) { + if (model.downloadPath === undefined) { console.log('Model not downloaded or not found') return } - console.log('Model downloaded at', this.modelPath) + console.log('Model downloaded at', model.downloadPath) - this._startServer() + if (this.inferenceProcess === null) await this._startServer() - this.isReady = true + switch (model.modelType) { + case ModelType.CODE_COMPLETION_INSERTION || ModelType.CODE_COMPLETION:{ + const res = await this._makeRequest('init_completion', { model_path: model.downloadPath }) + + if (res?.data?.status === "success") { + this.isReady = true + console.log('Completion Model initialized successfully') + } else { + this.isReady = false + console.error('Error initializing the model', res.data?.error) + } + break; + } + + case ModelType.GENERAL:{ + const res = await this._makeRequest('init', { model_path: model.downloadPath }) + + if (res.data?.status === "success") { + this.isReady = true + console.log('General Model initialized successfully') + } else { + this.isReady = false + console.error('Error initializing the model', res.data?.error) + } + break; + } + } + + this.stateTimer.start() + this.selectedModels.push(model) } catch (error) { console.error('Error initializing the model', error) this.isReady = false + InferenceManager.instance = null } } - async _downloadModel(model): Promise { + async _processStatus() { + const options = { headers: { 'Content-Type': 'application/json', } } + const state = await axios.get(this.inferenceURL+"/state", options) + + if (!state.data?.status) { + console.log('Inference server not running') + InferenceManager.instance = null + this.stateTimer.interval += this.stateTimer.interval + + if (this.stateTimer.interval >= 60000) { + // attempt to restart the server + console.log('Attempting to restart the server') + this.stopInferenceServer() + this._startServer() + this.stateTimer.interval = 20000 + } + } else { + // Server is running with successful request + // console.log('Inference server is running') + // console.log('completion is runnig', state.data?.completion) + // console.log('general is runnig', state.data?.general) + } + + } + async _downloadModel(model:IModel): Promise { if (this.modelCacheDir === undefined) { console.log('Model cache directory not provided') return @@ -79,10 +152,12 @@ export class InferenceManager implements ICompletions { const outputLocationPath = path.join(this.modelCacheDir, model.modelName); console.log('output location path is', outputLocationPath) if (fs.existsSync(outputLocationPath)) { - this.modelPath = outputLocationPath + model.downloadPath = outputLocationPath console.log('Model already exists in the output location', outputLocationPath); return; } + + console.log('Downloading model from', model.downloadUrl); // Make a HEAD request to get the file size const { headers } = await axios.head(model.downloadUrl); const totalSize = parseInt(headers['content-length'], 10); @@ -109,7 +184,7 @@ export class InferenceManager implements ICompletions { response.data.pipe(writer); this.event.emit('ready') - this.modelPath = outputLocationPath + model.downloadPath = outputLocationPath console.log('LLama Download complete'); return new Promise((resolve, reject) => { @@ -129,7 +204,7 @@ export class InferenceManager implements ICompletions { // Check if the file exists if (!fs.existsSync(serverPath)) { - return reject(new Error(`Python script not found at ${serverPath}`)); + return reject(new Error(`Inference server not found at ${serverPath}`)); } // Check file permissions @@ -139,8 +214,7 @@ export class InferenceManager implements ICompletions { return reject(new Error(`No execute permission on ${serverPath}`)); } - console.log('Running in non-pkg environment'); - const spawnArgs = ['5501', this.modelPath]; + const spawnArgs = [this.port]; console.log(`Spawning process: ${serverPath} ${spawnArgs.join(' ')}`); this.inferenceProcess = spawn(serverPath, spawnArgs); @@ -154,7 +228,11 @@ export class InferenceManager implements ICompletions { }); this.inferenceProcess.stderr.on('data', (data) => { - console.error(`Inference server: ${data}`); + console.error(`Inference log: ${data}`); + if (data.includes('Address already in use')) { + console.error(`Port ${this.port} is already in use. Please stop the existing server and try again`); + reject(new Error(`Port ${this.port} is already in use`)); + } resolve(); }); @@ -179,14 +257,14 @@ export class InferenceManager implements ICompletions { } } - private async _makeRequest(endpoint, payload){ + private async _makeInferenceRequest(endpoint, payload){ try { this.event.emit('onInference') const options = { headers: { 'Content-Type': 'application/json', } } const response = await axios.post(`${this.inferenceURL}/${endpoint}`, payload, options) this.event.emit('onInferenceDone') - if (response?.data?.generatedText) { + if (response.data?.generatedText) { return response.data.generatedText } else { return "" } } catch (error) { @@ -194,7 +272,56 @@ export class InferenceManager implements ICompletions { } } - async code_completion(context: any, params:IParams=completionParams): Promise { + private async _streamInferenceRequest(endpoint, payload){ + try { + this.event.emit('onInference') + const options = { headers: { 'Content-Type': 'application/json', } } + const response = await axios({ + method: 'post', + url: `${this.inferenceURL}/${endpoint}`, + data: payload, + headers: { + "Content-Type": "application/json", + "Accept": "text/event-stream", + } + , responseType: 'stream' }); + + response.data.on('data', (chunk: Buffer) => { + try { + const parsedData = JSON.parse(chunk.toString()); + if (parsedData.isGenerating) { + this.event.emit('onStreamResult', parsedData.generatedText); + } else { + return parsedData.generatedText + } + + } catch (error) { + console.error('Error parsing JSON:', error); + } + }); + + return "" // return empty string for now as payload already handled in event + } catch (error) { + console.error('Error making stream request to Inference server:', error.message); + } + finally { + this.event.emit('onInferenceDone') + } + } + + private async _makeRequest(endpoint, payload){ + try { + const options = { headers: { 'Content-Type': 'application/json', } } + const response = await axios.post(`${this.inferenceURL}/${endpoint}`, payload, options) + this.event.emit('onInferenceDone') + + return response + } catch (error) { + console.error('Error making request to Inference server:', error.message); + } + } + + async code_completion(context: any, params:IParams=CompletionParams): Promise { if (!this.isReady) { console.log('model not ready yet') return @@ -202,17 +329,63 @@ export class InferenceManager implements ICompletions { // as of now no prompt required const payload = { context_code: context, ...params } - return this._makeRequest('code_completion', payload) + return this._makeInferenceRequest('code_completion', payload) } - async code_insertion(msg_pfx: string, msg_sfx: string, params:IParams=insertionParams): Promise { + async code_insertion(msg_pfx: string, msg_sfx: string, params:IParams=InsertionParams): Promise { if (!this.isReady) { console.log('model not ready yet') return } const payload = { code_pfx:msg_pfx, code_sfx:msg_sfx, ...params } - return this._makeRequest('code_insertion', payload) + return this._makeInferenceRequest('code_insertion', payload) } + async code_generation(prompt: string, params:IParams=GenerationParams): Promise { + if (!this.isReady) { + console.log('model not ready yet') + return + } + return this._makeInferenceRequest('code_generation', { prompt, ...params }) + } + + async code_explaining(code:string, context:string, params:IParams=GenerationParams): Promise { + if (!this.isReady) { + console.log('model not ready yet') + return + } + if (GenerationParams.stream_result) { + return this._streamInferenceRequest('code_explaining', { code, context, ...params }) + } else { + return this._makeInferenceRequest('code_explaining', { code, context, ...params }) + } + } + + async error_explaining(prompt: string, params:IParams=GenerationParams): Promise{ + if (!this.isReady) { + console.log('model not ready yet') + return "" + } + if (GenerationParams.stream_result) { + return this._streamInferenceRequest('error_explaining', { prompt, ...params }) + } else { + return this._makeInferenceRequest('error_explaining', { prompt, ...params }) + } + } + + async solidity_answer(prompt: string, params:IParams=GenerationParams): Promise { + if (!this.isReady) { + console.log('model not ready yet') + return + } + if (GenerationParams.stream_result) { + return this._streamInferenceRequest('solidity_answer', { prompt, ...params }) + } else { + return this._makeInferenceRequest('solidity_answer', { prompt, ...params }) + } + } + + // kill dangling process making use of the port + } \ No newline at end of file diff --git a/apps/remixdesktop/src/plugins/remixAIDektop.ts b/apps/remixdesktop/src/plugins/remixAIDektop.ts index f99d9dd927..8a0175d6c3 100644 --- a/apps/remixdesktop/src/plugins/remixAIDektop.ts +++ b/apps/remixdesktop/src/plugins/remixAIDektop.ts @@ -5,6 +5,8 @@ import { Profile } from "@remixproject/plugin-utils" // use remix ai core import { InferenceManager } from "../lib/InferenceServerManager" import { cacheDir } from "../utils/config" +import { RemoteInferencer } from "../../../../libs/remix-ai-core/src/index" + // import { isE2E } from "../main"; const profile = { @@ -42,7 +44,7 @@ const clientProfile: Profile = { class RemixAIDesktopPluginClient extends ElectronBasePluginClient { readonly modelCacheDir: string = cacheDir - InferenceModel:InferenceManager = null + desktopInferencer:InferenceManager | RemoteInferencer = null constructor (webContentsId: number, profile: Profile){ console.log("loading the remix plugin client ........................") @@ -62,23 +64,56 @@ class RemixAIDesktopPluginClient extends ElectronBasePluginClient { this.emit('enabled') } - async initializeModelBackend(multitaskModel: any){ - if (this.InferenceModel === null) { - console.log('Initializing Inference model') - this.InferenceModel = InferenceManager.getInstance(multitaskModel, this.modelCacheDir) - if (!this.InferenceModel.isReady) this.InferenceModel.init() + async initializeModelBackend(local, generalModel?, completionModel?){ + if (local){ + console.log('Initializing Inference model locally') + this.desktopInferencer = new RemoteInferencer() } else { - console.log('Inference model already initialized') + if (!this.desktopInferencer){ + console.log('Initializing Inference model') + this.desktopInferencer = InferenceManager.getInstance(this.modelCacheDir) + } else { + console.log('Inference model already initialized') + } + if (this.desktopInferencer instanceof InferenceManager && generalModel) await this.desktopInferencer.init(generalModel) + if (this.desktopInferencer instanceof InferenceManager && completionModel) await this.desktopInferencer.init(completionModel) } + + // set event listeners + this.desktopInferencer.event.on('onStreamResult', (data) => { + this.emit('onStreamResult', data) + }) + this.desktopInferencer.event.on('onInference', () => { + this.emit('onInference') + }) + this.desktopInferencer.event.on('onInfrenceDone', () => { + this.emit('onInfrenceDone') + }) } async code_completion(context: any) { // use general purpose model - return this.InferenceModel.code_completion(context) + return this.desktopInferencer.code_completion(context) } async code_insertion(msg_pfx: string, msg_sfx: string) { - return this.InferenceModel.code_insertion(msg_pfx, msg_sfx) + return this.desktopInferencer.code_insertion(msg_pfx, msg_sfx) + } + + async code_generation(prompt: string) { + return this.desktopInferencer.code_generation(prompt) + } + + async code_explaining(code:string, context?:string) { + return this.desktopInferencer.code_explaining(code, context) + } + + async error_explaining(prompt: string) { + return this.desktopInferencer.error_explaining(prompt) + } + + async solidity_answer(prompt: string) { + return this.desktopInferencer.solidity_answer(prompt) } changemodel(newModel: any){ diff --git a/apps/vyper/src/app/utils/remix-client.tsx b/apps/vyper/src/app/utils/remix-client.tsx index f880d4f6a5..2e5720c03c 100644 --- a/apps/vyper/src/app/utils/remix-client.tsx +++ b/apps/vyper/src/app/utils/remix-client.tsx @@ -70,7 +70,7 @@ export class RemixClient extends PluginClient { ${message} can you explain why this error occurred and how to fix it? ` - await this.client.call('solcoder' as any, 'error_explaining', message) + await this.client.call('remixAI' as any, 'error_explaining', message) } catch (err) { console.error('unable to askGpt') console.error(err) diff --git a/libs/remix-ai-core/src/index.ts b/libs/remix-ai-core/src/index.ts index 68e7b9e7cd..ecc00d4f65 100644 --- a/libs/remix-ai-core/src/index.ts +++ b/libs/remix-ai-core/src/index.ts @@ -1,17 +1,18 @@ 'use strict' import { IModel, IModelResponse, IModelRequest, InferenceModel, ICompletions, - IParams, ChatEntry, AIRequestType, RemoteBackendOPModel } from './types/types' + IParams, ChatEntry, AIRequestType, IRemoteModel, + RemoteBackendOPModel, IStreamResponse } from './types/types' import { ModelType } from './types/constants' -import { DefaultModels } from './types/models' +import { DefaultModels, InsertionParams, CompletionParams, GenerationParams } from './types/models' import { getCompletionPrompt, getInsertionPrompt } from './prompts/completionPrompts' import { PromptBuilder } from './prompts/promptBuilder' import { RemoteInferencer } from './inferencers/remote/remoteInference' export { IModel, IModelResponse, IModelRequest, InferenceModel, - ModelType, DefaultModels, ICompletions, IParams, - getCompletionPrompt, getInsertionPrompt, - RemoteInferencer, + ModelType, DefaultModels, ICompletions, IParams, IRemoteModel, + getCompletionPrompt, getInsertionPrompt, IStreamResponse, + RemoteInferencer, InsertionParams, CompletionParams, GenerationParams, ChatEntry, AIRequestType, RemoteBackendOPModel, PromptBuilder } \ No newline at end of file diff --git a/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts b/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts index acd95afae4..4e3f911916 100644 --- a/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts +++ b/libs/remix-ai-core/src/inferencers/remote/remoteInference.ts @@ -1,6 +1,7 @@ import { ICompletions, IParams, ChatEntry, AIRequestType, RemoteBackendOPModel } from "../../types/types"; import { PromptBuilder } from "../../prompts/promptBuilder"; import axios from "axios"; +import EventEmitter from "events"; const defaultErrorMessage = `Unable to get a response from AI server` @@ -10,13 +11,13 @@ export class RemoteInferencer implements ICompletions { solgpt_chat_history:ChatEntry[] max_history = 7 model_op = RemoteBackendOPModel.DEEPSEEK - mainPlugin = null + event: EventEmitter - constructor(plugin, apiUrl?:string, completionUrl?:string) { + constructor(apiUrl?:string, completionUrl?:string) { this.api_url = apiUrl!==undefined ? apiUrl: "https://solcoder.remixproject.org" this.completion_url = completionUrl!==undefined ? completionUrl : "https://completion.remixproject.org" this.solgpt_chat_history = [] - this.mainPlugin = plugin + this.event = new EventEmitter() } private pushChatHistory(prompt, result){ @@ -26,7 +27,7 @@ export class RemoteInferencer implements ICompletions { } private async _makeRequest(data, rType:AIRequestType){ - this.mainPlugin.emit("aiInfering") + this.event.emit("onInference") const requesURL = rType === AIRequestType.COMPLETION ? this.completion_url : this.api_url console.log("requesting on ", requesURL, rType, data.data[1]) @@ -45,27 +46,61 @@ export class RemoteInferencer implements ICompletions { if (result.statusText === "OK") return result.data.data[0] else { - this.mainPlugin.call('terminal', 'log', { type: 'aitypewriterwarning', value: defaultErrorMessage }) - return "" + return defaultErrorMessage } case AIRequestType.GENERAL: if (result.statusText === "OK") { const resultText = result.data.data[0] - this.mainPlugin.call('terminal', 'log', { type: 'aitypewriterwarning', value: resultText }) this.pushChatHistory(prompt, resultText) + return resultText } else { - this.mainPlugin.call('terminal', 'log', { type: 'aitypewriterwarning', value: defaultErrorMessage }) + return defaultErrorMessage } - break } } catch (e) { - this.mainPlugin.call('terminal', 'log', { type: 'aitypewriterwarning', value: defaultErrorMessage }) this.solgpt_chat_history = [] - return "" + return e } finally { - this.mainPlugin.emit("aiInferingDone") + this.event.emit("onInferenceDone") + } + } + + private async _streamInferenceRequest(data, rType:AIRequestType){ + try { + this.event.emit('onInference') + const requesURL = rType === AIRequestType.COMPLETION ? this.completion_url : this.api_url + const options = { headers: { 'Content-Type': 'application/json', "Accept": "text/event-stream" } } + const response = await axios({ + method: 'post', + url: rType === AIRequestType.COMPLETION ? this.completion_url : this.api_url, + data: data, + headers: { 'Content-Type': 'application/json', "Accept": "text/event-stream" }, + responseType: 'stream' + }); + + response.data.on('data', (chunk: Buffer) => { + try { + const parsedData = JSON.parse(chunk.toString()); + if (parsedData.isGenerating) { + this.event.emit('onStreamResult', parsedData.generatedText); + } else { + return parsedData.generatedText + } + + } catch (error) { + console.error('Error parsing JSON:', error); + } + + }); + + return "" // return empty string for now as handled in event + } catch (error) { + console.error('Error making stream request to Inference server:', error.message); + } + finally { + this.event.emit('onInferenceDone') } } @@ -90,20 +125,17 @@ export class RemoteInferencer implements ICompletions { } async solidity_answer(prompt): Promise { - this.mainPlugin.call('terminal', 'log', { type: 'aitypewriterwarning', value: `\n\nWaiting for RemixAI answer...` }) const main_prompt = this._build_solgpt_promt(prompt) const payload = { "data":[main_prompt, "solidity_answer", false,2000,0.9,0.8,50]} return this._makeRequest(payload, AIRequestType.GENERAL) } async code_explaining(prompt, context:string=""): Promise { - this.mainPlugin.call('terminal', 'log', { type: 'aitypewriterwarning', value: `\n\nWaiting for RemixAI answer...` }) const payload = { "data":[prompt, "code_explaining", false,2000,0.9,0.8,50, context]} return this._makeRequest(payload, AIRequestType.GENERAL) } async error_explaining(prompt): Promise { - this.mainPlugin.call('terminal', 'log', { type: 'aitypewriterwarning', value: `\n\nWaiting for RemixAI answer...` }) const payload = { "data":[prompt, "error_explaining", false,2000,0.9,0.8,50]} return this._makeRequest(payload, AIRequestType.GENERAL) } diff --git a/libs/remix-ai-core/src/types/models.ts b/libs/remix-ai-core/src/types/models.ts index 8efe61de9e..ab53a3cc45 100644 --- a/libs/remix-ai-core/src/types/models.ts +++ b/libs/remix-ai-core/src/types/models.ts @@ -2,7 +2,7 @@ // create a function getModels returning a list of all supported models // create a function getModel returning a model by its name -import { IModel } from './types'; +import { IModel, IParams } from './types'; import { ModelType } from './constants'; const DefaultModels = (): IModel[] => { @@ -47,7 +47,17 @@ const DefaultModels = (): IModel[] => { modelType: ModelType.CODE_COMPLETION_INSERTION, modelReqs: { backend: 'llamacpp', minSysMemory: 2, GPURequired: false, MinGPUVRAM: 2 } }; - return [model1, model2, model3, model4, model5]; + + const model6: IModel = { + name: 'DeepSeek', + task: 'text-generation', + modelName: 'DeepSeek-Coder-V2-Lite-Base.Q2_K.gguf', + downloadUrl: 'https://huggingface.co/QuantFactory/DeepSeek-Coder-V2-Lite-Base-GGUF/resolve/main/DeepSeek-Coder-V2-Lite-Base.Q2_K.gguf?download=true', + modelType: ModelType.GENERAL, + modelReqs: { backend: 'llamacpp', minSysMemory: 2, GPURequired: false, MinGPUVRAM: 8 } + }; + + return [model1, model2, model3, model4, model5, model6]; } const getModel = async (name: string): Promise => { @@ -58,4 +68,26 @@ const loadModel = async (modelname: string): Promise => { console.log(`Loading model ${modelname}`); } -export { DefaultModels } \ No newline at end of file +const CompletionParams:IParams = { + temperature: 0.8, + topK: 40, + topP: 0.92, + max_new_tokens: 15, +} + +const InsertionParams:IParams = { + temperature: 0.8, + topK: 40, + topP: 0.92, + max_new_tokens: 150, +} + +const GenerationParams:IParams = { + temperature: 0.5, + topK: 40, + topP: 0.92, + max_new_tokens: 2000, + stream_result: true, +} + +export { DefaultModels, CompletionParams, InsertionParams, GenerationParams } \ No newline at end of file diff --git a/libs/remix-ai-core/src/types/types.ts b/libs/remix-ai-core/src/types/types.ts index b94c3a6e78..5992c4a641 100644 --- a/libs/remix-ai-core/src/types/types.ts +++ b/libs/remix-ai-core/src/types/types.ts @@ -17,6 +17,11 @@ export interface IModel { modelName: string; modelType: ModelType; modelReqs: IModelRequirements; + downloadPath?: string; +} +export interface IRemoteModel { + completionUrl: string; + apiUrl: string; } export interface IModelResponse { @@ -26,6 +31,11 @@ export interface IModelResponse { model: IModel; } +export interface IStreamResponse { + generatedText: string; + isGenerating: boolean; +} + export interface IModelRequest { input: string; model: IModel; diff --git a/libs/remix-ui/editor/src/lib/remix-ui-editor.tsx b/libs/remix-ui/editor/src/lib/remix-ui-editor.tsx index 7805223b87..ad70a135c9 100644 --- a/libs/remix-ui/editor/src/lib/remix-ui-editor.tsx +++ b/libs/remix-ui/editor/src/lib/remix-ui-editor.tsx @@ -737,7 +737,7 @@ export const EditorUI = (props: EditorUIProps) => { const file = await props.plugin.call('fileManager', 'getCurrentFile') const content = await props.plugin.call('fileManager', 'readFile', file) const message = intl.formatMessage({ id: 'editor.generateDocumentationByAI' }, { content, currentFunction: currentFunction.current }) - const cm = await props.plugin.call('solcoder', 'code_explaining', message) + const cm = await props.plugin.call('remixAI', 'code_explaining', message) const natSpecCom = "\n" + extractNatspecComments(cm) const cln = await props.plugin.call('codeParser', "getLineColumnOfNode", currenFunctionNode) @@ -773,7 +773,7 @@ export const EditorUI = (props: EditorUIProps) => { }, ]); - _paq.push(['trackEvent', 'ai', 'solcoder', 'generateDocumentation']) + _paq.push(['trackEvent', 'ai', 'remixAI', 'generateDocumentation']) }, } @@ -791,8 +791,8 @@ export const EditorUI = (props: EditorUIProps) => { const file = await props.plugin.call('fileManager', 'getCurrentFile') const content = await props.plugin.call('fileManager', 'readFile', file) const message = intl.formatMessage({ id: 'editor.explainFunctionByAI' }, { content, currentFunction: currentFunction.current }) - await props.plugin.call('solcoder', 'code_explaining', message, content) - _paq.push(['trackEvent', 'ai', 'solcoder', 'explainFunction']) + await props.plugin.call('remixAI', 'code_explaining', message, content) + _paq.push(['trackEvent', 'ai', 'remixAI', 'explainFunction']) }, } @@ -811,8 +811,8 @@ export const EditorUI = (props: EditorUIProps) => { const content = await props.plugin.call('fileManager', 'readFile', file) const selectedCode = editor.getModel().getValueInRange(editor.getSelection()) - await props.plugin.call('solcoder', 'code_explaining', selectedCode, content) - _paq.push(['trackEvent', 'ai', 'solcoder', 'explainFunction']) + await props.plugin.call('remixAI', 'code_explaining', selectedCode, content) + _paq.push(['trackEvent', 'ai', 'remixAI', 'explainFunction']) }, } diff --git a/libs/remix-ui/remix-ai/src/lib/components/Default.tsx b/libs/remix-ui/remix-ai/src/lib/components/Default.tsx index 9052a2b617..ccc179b635 100644 --- a/libs/remix-ui/remix-ai/src/lib/components/Default.tsx +++ b/libs/remix-ui/remix-ai/src/lib/components/Default.tsx @@ -6,6 +6,20 @@ export const Default = (props) => { const [searchText, setSearchText] = useState(''); const [resultText, setResultText] = useState(''); const pluginName = 'remixAI' + const appendText = (newText) => { + setResultText(resultText => resultText + newText); + } + + useEffect(() => { + const handleResultReady = async (e) => { + appendText(e); + }; + if (props.plugin.isOnDesktop ) { + props.plugin.on(props.plugin.remixDesktopPluginName, 'onStreamResult', (value) => { + handleResultReady(value); + }) + } + }, []) return (
@@ -46,19 +60,17 @@ export const Default = (props) => { rows={10} cols={50} placeholder="Results..." + onChange={(e) => { + console.log('resultText changed', e.target.value) + setResultText(e.target.value)} + } value={resultText} readOnly />
diff --git a/libs/remix-ui/renderer/src/lib/renderer.tsx b/libs/remix-ui/renderer/src/lib/renderer.tsx index 006de70049..984f180e0f 100644 --- a/libs/remix-ui/renderer/src/lib/renderer.tsx +++ b/libs/remix-ui/renderer/src/lib/renderer.tsx @@ -75,8 +75,8 @@ export const Renderer = ({ message, opt = {}, plugin }: RendererProps) => { try { const content = await plugin.call('fileManager', 'readFile', editorOptions.errFile) const message = intl.formatMessage({ id: 'solidity.openaigptMessage' }, { content, messageText }) - await plugin.call('solcoder', 'error_explaining', message) - _paq.push(['trackEvent', 'ai', 'solcoder', 'error_explaining_SolidityError']) + await plugin.call('remixAI', 'error_explaining', message) + _paq.push(['trackEvent', 'ai', 'remixAI', 'error_explaining_SolidityError']) } catch (err) { console.error('unable to askGtp') console.error(err) diff --git a/libs/remix-ui/settings/src/lib/remix-ui-settings.tsx b/libs/remix-ui/settings/src/lib/remix-ui-settings.tsx index 050cb9d539..63296e4955 100644 --- a/libs/remix-ui/settings/src/lib/remix-ui-settings.tsx +++ b/libs/remix-ui/settings/src/lib/remix-ui-settings.tsx @@ -462,7 +462,7 @@ export const RemixUiSettings = (props: RemixUiSettingsProps) => { role='link' onClick={()=>{ window.open("https://remix-ide.readthedocs.io/en/latest/ai.html") - _paq.push(['trackEvent', 'ai', 'solcoder', 'documentation']) + _paq.push(['trackEvent', 'ai', 'remixAI', 'documentation']) }} > diff --git a/libs/remix-ui/tabs/src/lib/remix-ui-tabs.tsx b/libs/remix-ui/tabs/src/lib/remix-ui-tabs.tsx index f5c3a668c0..79c3c71a36 100644 --- a/libs/remix-ui/tabs/src/lib/remix-ui-tabs.tsx +++ b/libs/remix-ui/tabs/src/lib/remix-ui-tabs.tsx @@ -251,9 +251,9 @@ export const TabsUI = (props: TabsUIProps) => { const content = await props.plugin.call('fileManager', 'readFile', path) if (tabsState.currentExt === 'sol') { setExplaining(true) - await props.plugin.call('solcoder', 'code_explaining', content) + await props.plugin.call('remixAI', 'code_explaining', content) setExplaining(false) - _paq.push(['trackEvent', 'ai', 'solcoder', 'explain_file']) + _paq.push(['trackEvent', 'ai', 'remixAI', 'explain_file']) } }} > @@ -283,7 +283,7 @@ export const TabsUI = (props: TabsUIProps) => { onClick={async () => { await props.plugin.call('settings', 'updateCopilotChoice', !ai_switch) setAI_switch(!ai_switch) - ai_switch ? _paq.push(['trackEvent', 'ai', 'solcoder', 'copilot_enabled']) : _paq.push(['trackEvent', 'ai', 'solcoder', 'copilot_disabled']) + ai_switch ? _paq.push(['trackEvent', 'ai', 'remixAI', 'copilot_enabled']) : _paq.push(['trackEvent', 'ai', 'remixAI', 'copilot_disabled']) }} > diff --git a/libs/remix-ui/terminal/src/lib/remix-ui-terminal.tsx b/libs/remix-ui/terminal/src/lib/remix-ui-terminal.tsx index 9304f56d3d..5f006217ff 100644 --- a/libs/remix-ui/terminal/src/lib/remix-ui-terminal.tsx +++ b/libs/remix-ui/terminal/src/lib/remix-ui-terminal.tsx @@ -238,12 +238,12 @@ export const RemixUiTerminal = (props: RemixUiTerminalProps) => { // TODO: rm gpt or redirect gpt to sol-pgt } else if (script.trim().startsWith('gpt')) { call('terminal', 'log',{ type: 'warn', value: `> ${script}` }) - await call('solcoder', 'solidity_answer', script) - _paq.push(['trackEvent', 'ai', 'solcoder', 'askFromTerminal']) + await call('remixAI', 'solidity_answer', script) + _paq.push(['trackEvent', 'ai', 'remixAI', 'askFromTerminal']) } else if (script.trim().startsWith('sol-gpt')) { call('terminal', 'log',{ type: 'warn', value: `> ${script}` }) - await call('solcoder', 'solidity_answer', script) - _paq.push(['trackEvent', 'ai', 'solcoder', 'askFromTerminal']) + await call('remixAI', 'solidity_answer', script) + _paq.push(['trackEvent', 'ai', 'remixAI', 'askFromTerminal']) } else { await call('scriptRunner', 'execute', script) }