databatcher

apps/remix-ide/src/app/files/electronProvider.ts

@@ -17,30 +17,42 @@ export class ElectronProvider extends FileProvider {
 
   async init() {
     this._appManager.on('fs', 'change', async (event, path) => {
-      switch (event) {
-      case 'add':
-        this.event.emit('fileAdded', path)
-        break
-      case 'unlink':
-        this.event.emit('fileRemoved', path)
-        break
-      case 'change':
-        this.get(path, (_error, content) => {
-          this.event.emit('fileExternallyChanged', path, content, false)
-        })
-        break
-      case 'rename':
-        this.event.emit('fileRenamed', path)
-        break
-      case 'addDir':
-        this.event.emit('folderAdded', path)
-        break
-      case 'unlinkDir':
-        this.event.emit('fileRemoved', path)
+      this.handleEvent(event, path)
+    })
+    this._appManager.on('fs', 'eventGroup', async (data) => {
+      console.log('eventGroup', data)
+      for (const event of data) {
+        console.log('event', event)
+        this.handleEvent(event.payload[0], event.payload[1])
       }
     })
   }
 
+  handleEvent = (event, path) => {
+    switch (event) {
+    case 'add':
+      this.event.emit('fileAdded', path)
+      break
+    case 'unlink':
+      this.event.emit('fileRemoved', path)
+      break
+    case 'change':
+      this.get(path, (_error, content) => {
+        this.event.emit('fileExternallyChanged', path, content, false)
+      })
+      break
+    case 'rename':
+      this.event.emit('fileRenamed', path)
+      break
+    case 'addDir':
+      this.event.emit('folderAdded', path)
+      break
+    case 'unlinkDir':
+      this.event.emit('fileRemoved', path)
+    }
+  }
+
+
   // isDirectory is already included
   // this is a more efficient version of the default implementation
   async resolveDirectory(path, cb) {

apps/remixdesktop/src/plugins/fsPlugin.ts

@@ -7,6 +7,9 @@ import {createWindow, isPackaged} from '../main'
 import {writeConfig} from '../utils/config'
 import path from 'path'
 import {customAction} from '@remixproject/plugin-api'
+import { PluginEventDataBatcher } from '../utils/pluginEventDataBatcher'
+
+
 
 const profile: Profile = {
   displayName: 'fs',
@@ -87,6 +90,7 @@ class FSPluginClient extends ElectronBasePluginClient {
   workingDir: string = ''
   trackDownStreamUpdate: Record<string, string> = {}
   expandedPaths: string[] = ['.']
+  dataBatcher: PluginEventDataBatcher
 
   constructor(webContentsId: number, profile: Profile) {
     super(webContentsId, profile)
@@ -99,6 +103,11 @@ class FSPluginClient extends ElectronBasePluginClient {
         await this.closeWatch()
       })
     })
+    this.dataBatcher = new PluginEventDataBatcher(webContentsId)
+    this.dataBatcher.on('flush', (data: any) => {
+      console.log('flush', data)
+      this.emit('eventGroup', data)
+    })
   }
 
   // best for non recursive
@@ -201,7 +210,7 @@ class FSPluginClient extends ElectronBasePluginClient {
         this.expandedPaths = ['.', ...paths] // add root
         console.log(Object.keys(this.watchers))
         paths = paths.map((path) => this.fixPath(path))
-        for (let path of paths) {
+        for (const path of paths) {
           if (!Object.keys(this.watchers).includes(path)) {
             this.watchers[path] = await this.watcherInit(path)
             console.log('added watcher', path)
@@ -263,9 +272,8 @@ class FSPluginClient extends ElectronBasePluginClient {
           const dirname = path.dirname(pathWithoutPrefix)
           if (this.expandedPaths.includes(dirname) || this.expandedPaths.includes(pathWithoutPrefix)) {
             console.log('emitting', eventName, pathWithoutPrefix, this.expandedPaths)
-            this.emit('change', eventName, pathWithoutPrefix)
+            this.dataBatcher.write('change', eventName, pathWithoutPrefix)
           }
-          this.emit('change', eventName, pathWithoutPrefix)
         } catch (e) {
           console.log('error emitting change', e)
         }
@@ -276,7 +284,8 @@ class FSPluginClient extends ElectronBasePluginClient {
         console.log('check emitting', eventName, pathWithoutPrefix, this.expandedPaths, dirname)
         if (this.expandedPaths.includes(dirname) || this.expandedPaths.includes(pathWithoutPrefix)) {
           console.log('emitting', eventName, pathWithoutPrefix, this.expandedPaths)
-          this.emit('change', eventName, pathWithoutPrefix)
+          //this.emit('change', eventName, pathWithoutPrefix)
+          this.dataBatcher.write('change', eventName, pathWithoutPrefix)
         }
       } catch (e) {
         console.log('error emitting change', e)

apps/remixdesktop/src/utils/pluginEventDataBatcher.ts

@@ -0,0 +1,62 @@
+import {EventEmitter} from 'events';
+import { StringDecoder } from 'string_decoder';
+// Max duration to batch session data before sending it to the renderer process.
+const BATCH_DURATION_MS = 16;
+
+// Max size of a session data batch. Note that this value can be exceeded by ~4k
+// (chunk sizes seem to be 4k at the most)
+const BATCH_MAX_SIZE = 200 * 1024;
+
+// Data coming from the pty is sent to the renderer process for further
+// vt parsing and rendering. This class batches data to minimize the number of
+// IPC calls. It also reduces GC pressure and CPU cost: each chunk is prefixed
+// with the window ID which is then stripped on the renderer process and this
+// overhead is reduced with batching.
+export class PluginEventDataBatcher extends EventEmitter {
+  uid: number;
+  decoder: StringDecoder;
+  data!: any[]
+  timeout!: NodeJS.Timeout | null;
+  constructor(uid: number) {
+    super();
+    this.uid = uid;
+    this.decoder = new StringDecoder('utf8');
+
+    this.reset();
+  }
+
+  reset() {
+    this.data = [];
+    this.timeout = null;
+  }
+
+  write(key: string, ...payload: any): void {
+    if (this.data.length >= BATCH_MAX_SIZE) {
+      // We've reached the max batch size. Flush it and start another one
+      if (this.timeout) {
+        clearTimeout(this.timeout);
+        this.timeout = null;
+      }
+      this.flush();
+    }
+
+    this.data.push({
+      key,
+      payload,
+    });
+
+    console.log('data', this.data)
+
+    if (!this.timeout) {
+      this.timeout = setTimeout(() => this.flush(), BATCH_DURATION_MS);
+    }
+  }
+
+  flush() {
+    // Reset before emitting to allow for potential reentrancy
+    const data = this.data;
+    this.reset();
+
+    this.emit('flush', data);
+  }
+}

libs/remix-ui/workspace/src/lib/components/file-explorer.tsx

@@ -398,9 +398,9 @@ export const FileExplorer = (props: FileExplorerProps) => {
   return (
 
     <div ref={treeRef} tabIndex={0} style={{ outline: 'none',
-    display: 'flex',
-    flexDirection: 'column',
-    height: '100%',
+      display: 'flex',
+      flexDirection: 'column',
+      height: '100%',
     }}>
       <TreeView id="treeView">
         <li key={`treeViewLiMenu`} data-id={`treeViewLiMenu`} className="li_tv">
@@ -426,19 +426,19 @@ export const FileExplorer = (props: FileExplorerProps) => {
           </div>
         </li>
         <div style={{flexGrow: 2}}>
-        <div>
-        <FlatTree
-          treeRef={treeRef}
-          handleTreeClick={handleTreeClick}
-          focusEdit={state.focusEdit}
-          focusElement={props.focusElement}
-          focusContext={state.focusContext}
-          editModeOff={editModeOff}
-          files={files}
-          expandPath={props.expandPath}
-          handleContextMenu={handleContextMenu} 
-        />
-        </div>
+          <div>
+            <FlatTree
+              treeRef={treeRef}
+              handleTreeClick={handleTreeClick}
+              focusEdit={state.focusEdit}
+              focusElement={props.focusElement}
+              focusContext={state.focusContext}
+              editModeOff={editModeOff}
+              files={files}
+              expandPath={props.expandPath}
+              handleContextMenu={handleContextMenu} 
+            />
+          </div>
         </div>
           
         <div className='d-block w-100 pb-4 mb-4'></div>