From 653ff6aaf62a8fafb34bb99c4d53f1221b213d48 Mon Sep 17 00:00:00 2001 From: bunsenstraat Date: Mon, 25 Dec 2023 18:01:54 +0100 Subject: [PATCH] databatcher --- .../src/app/files/electronProvider.ts | 52 ++++++++++------ apps/remixdesktop/src/plugins/fsPlugin.ts | 17 +++-- .../src/utils/pluginEventDataBatcher.ts | 62 +++++++++++++++++++ .../src/lib/components/file-explorer.tsx | 32 +++++----- 4 files changed, 123 insertions(+), 40 deletions(-) create mode 100644 apps/remixdesktop/src/utils/pluginEventDataBatcher.ts diff --git a/apps/remix-ide/src/app/files/electronProvider.ts b/apps/remix-ide/src/app/files/electronProvider.ts index 2c59c8a25c..db7f399ba6 100644 --- a/apps/remix-ide/src/app/files/electronProvider.ts +++ b/apps/remix-ide/src/app/files/electronProvider.ts @@ -17,30 +17,42 @@ export class ElectronProvider extends FileProvider { async init() { this._appManager.on('fs', 'change', async (event, path) => { - switch (event) { - case 'add': - this.event.emit('fileAdded', path) - break - case 'unlink': - this.event.emit('fileRemoved', path) - break - case 'change': - this.get(path, (_error, content) => { - this.event.emit('fileExternallyChanged', path, content, false) - }) - break - case 'rename': - this.event.emit('fileRenamed', path) - break - case 'addDir': - this.event.emit('folderAdded', path) - break - case 'unlinkDir': - this.event.emit('fileRemoved', path) + this.handleEvent(event, path) + }) + this._appManager.on('fs', 'eventGroup', async (data) => { + console.log('eventGroup', data) + for (const event of data) { + console.log('event', event) + this.handleEvent(event.payload[0], event.payload[1]) } }) } + handleEvent = (event, path) => { + switch (event) { + case 'add': + this.event.emit('fileAdded', path) + break + case 'unlink': + this.event.emit('fileRemoved', path) + break + case 'change': + this.get(path, (_error, content) => { + this.event.emit('fileExternallyChanged', path, content, false) + }) + break + case 'rename': + this.event.emit('fileRenamed', path) + break + case 'addDir': + this.event.emit('folderAdded', path) + break + case 'unlinkDir': + this.event.emit('fileRemoved', path) + } + } + + // isDirectory is already included // this is a more efficient version of the default implementation async resolveDirectory(path, cb) { diff --git a/apps/remixdesktop/src/plugins/fsPlugin.ts b/apps/remixdesktop/src/plugins/fsPlugin.ts index 78b376473f..b1c37bf447 100644 --- a/apps/remixdesktop/src/plugins/fsPlugin.ts +++ b/apps/remixdesktop/src/plugins/fsPlugin.ts @@ -7,6 +7,9 @@ import {createWindow, isPackaged} from '../main' import {writeConfig} from '../utils/config' import path from 'path' import {customAction} from '@remixproject/plugin-api' +import { PluginEventDataBatcher } from '../utils/pluginEventDataBatcher' + + const profile: Profile = { displayName: 'fs', @@ -87,6 +90,7 @@ class FSPluginClient extends ElectronBasePluginClient { workingDir: string = '' trackDownStreamUpdate: Record = {} expandedPaths: string[] = ['.'] + dataBatcher: PluginEventDataBatcher constructor(webContentsId: number, profile: Profile) { super(webContentsId, profile) @@ -99,6 +103,11 @@ class FSPluginClient extends ElectronBasePluginClient { await this.closeWatch() }) }) + this.dataBatcher = new PluginEventDataBatcher(webContentsId) + this.dataBatcher.on('flush', (data: any) => { + console.log('flush', data) + this.emit('eventGroup', data) + }) } // best for non recursive @@ -201,7 +210,7 @@ class FSPluginClient extends ElectronBasePluginClient { this.expandedPaths = ['.', ...paths] // add root console.log(Object.keys(this.watchers)) paths = paths.map((path) => this.fixPath(path)) - for (let path of paths) { + for (const path of paths) { if (!Object.keys(this.watchers).includes(path)) { this.watchers[path] = await this.watcherInit(path) console.log('added watcher', path) @@ -263,9 +272,8 @@ class FSPluginClient extends ElectronBasePluginClient { const dirname = path.dirname(pathWithoutPrefix) if (this.expandedPaths.includes(dirname) || this.expandedPaths.includes(pathWithoutPrefix)) { console.log('emitting', eventName, pathWithoutPrefix, this.expandedPaths) - this.emit('change', eventName, pathWithoutPrefix) + this.dataBatcher.write('change', eventName, pathWithoutPrefix) } - this.emit('change', eventName, pathWithoutPrefix) } catch (e) { console.log('error emitting change', e) } @@ -276,7 +284,8 @@ class FSPluginClient extends ElectronBasePluginClient { console.log('check emitting', eventName, pathWithoutPrefix, this.expandedPaths, dirname) if (this.expandedPaths.includes(dirname) || this.expandedPaths.includes(pathWithoutPrefix)) { console.log('emitting', eventName, pathWithoutPrefix, this.expandedPaths) - this.emit('change', eventName, pathWithoutPrefix) + //this.emit('change', eventName, pathWithoutPrefix) + this.dataBatcher.write('change', eventName, pathWithoutPrefix) } } catch (e) { console.log('error emitting change', e) diff --git a/apps/remixdesktop/src/utils/pluginEventDataBatcher.ts b/apps/remixdesktop/src/utils/pluginEventDataBatcher.ts new file mode 100644 index 0000000000..2a380b2053 --- /dev/null +++ b/apps/remixdesktop/src/utils/pluginEventDataBatcher.ts @@ -0,0 +1,62 @@ +import {EventEmitter} from 'events'; +import { StringDecoder } from 'string_decoder'; +// Max duration to batch session data before sending it to the renderer process. +const BATCH_DURATION_MS = 16; + +// Max size of a session data batch. Note that this value can be exceeded by ~4k +// (chunk sizes seem to be 4k at the most) +const BATCH_MAX_SIZE = 200 * 1024; + +// Data coming from the pty is sent to the renderer process for further +// vt parsing and rendering. This class batches data to minimize the number of +// IPC calls. It also reduces GC pressure and CPU cost: each chunk is prefixed +// with the window ID which is then stripped on the renderer process and this +// overhead is reduced with batching. +export class PluginEventDataBatcher extends EventEmitter { + uid: number; + decoder: StringDecoder; + data!: any[] + timeout!: NodeJS.Timeout | null; + constructor(uid: number) { + super(); + this.uid = uid; + this.decoder = new StringDecoder('utf8'); + + this.reset(); + } + + reset() { + this.data = []; + this.timeout = null; + } + + write(key: string, ...payload: any): void { + if (this.data.length >= BATCH_MAX_SIZE) { + // We've reached the max batch size. Flush it and start another one + if (this.timeout) { + clearTimeout(this.timeout); + this.timeout = null; + } + this.flush(); + } + + this.data.push({ + key, + payload, + }); + + console.log('data', this.data) + + if (!this.timeout) { + this.timeout = setTimeout(() => this.flush(), BATCH_DURATION_MS); + } + } + + flush() { + // Reset before emitting to allow for potential reentrancy + const data = this.data; + this.reset(); + + this.emit('flush', data); + } +} \ No newline at end of file diff --git a/libs/remix-ui/workspace/src/lib/components/file-explorer.tsx b/libs/remix-ui/workspace/src/lib/components/file-explorer.tsx index 7d54e76a4e..09db5cb396 100644 --- a/libs/remix-ui/workspace/src/lib/components/file-explorer.tsx +++ b/libs/remix-ui/workspace/src/lib/components/file-explorer.tsx @@ -398,9 +398,9 @@ export const FileExplorer = (props: FileExplorerProps) => { return (
  • @@ -426,19 +426,19 @@ export const FileExplorer = (props: FileExplorerProps) => {
    • -
    - -
    +
    + +