databatcher

flattentree
bunsenstraat 11 months ago
parent c5cfb5cb27
commit 653ff6aaf6
  1. 52
      apps/remix-ide/src/app/files/electronProvider.ts
  2. 17
      apps/remixdesktop/src/plugins/fsPlugin.ts
  3. 62
      apps/remixdesktop/src/utils/pluginEventDataBatcher.ts
  4. 32
      libs/remix-ui/workspace/src/lib/components/file-explorer.tsx

@ -17,30 +17,42 @@ export class ElectronProvider extends FileProvider {
async init() {
this._appManager.on('fs', 'change', async (event, path) => {
switch (event) {
case 'add':
this.event.emit('fileAdded', path)
break
case 'unlink':
this.event.emit('fileRemoved', path)
break
case 'change':
this.get(path, (_error, content) => {
this.event.emit('fileExternallyChanged', path, content, false)
})
break
case 'rename':
this.event.emit('fileRenamed', path)
break
case 'addDir':
this.event.emit('folderAdded', path)
break
case 'unlinkDir':
this.event.emit('fileRemoved', path)
this.handleEvent(event, path)
})
this._appManager.on('fs', 'eventGroup', async (data) => {
console.log('eventGroup', data)
for (const event of data) {
console.log('event', event)
this.handleEvent(event.payload[0], event.payload[1])
}
})
}
handleEvent = (event, path) => {
switch (event) {
case 'add':
this.event.emit('fileAdded', path)
break
case 'unlink':
this.event.emit('fileRemoved', path)
break
case 'change':
this.get(path, (_error, content) => {
this.event.emit('fileExternallyChanged', path, content, false)
})
break
case 'rename':
this.event.emit('fileRenamed', path)
break
case 'addDir':
this.event.emit('folderAdded', path)
break
case 'unlinkDir':
this.event.emit('fileRemoved', path)
}
}
// isDirectory is already included
// this is a more efficient version of the default implementation
async resolveDirectory(path, cb) {

@ -7,6 +7,9 @@ import {createWindow, isPackaged} from '../main'
import {writeConfig} from '../utils/config'
import path from 'path'
import {customAction} from '@remixproject/plugin-api'
import { PluginEventDataBatcher } from '../utils/pluginEventDataBatcher'
const profile: Profile = {
displayName: 'fs',
@ -87,6 +90,7 @@ class FSPluginClient extends ElectronBasePluginClient {
workingDir: string = ''
trackDownStreamUpdate: Record<string, string> = {}
expandedPaths: string[] = ['.']
dataBatcher: PluginEventDataBatcher
constructor(webContentsId: number, profile: Profile) {
super(webContentsId, profile)
@ -99,6 +103,11 @@ class FSPluginClient extends ElectronBasePluginClient {
await this.closeWatch()
})
})
this.dataBatcher = new PluginEventDataBatcher(webContentsId)
this.dataBatcher.on('flush', (data: any) => {
console.log('flush', data)
this.emit('eventGroup', data)
})
}
// best for non recursive
@ -201,7 +210,7 @@ class FSPluginClient extends ElectronBasePluginClient {
this.expandedPaths = ['.', ...paths] // add root
console.log(Object.keys(this.watchers))
paths = paths.map((path) => this.fixPath(path))
for (let path of paths) {
for (const path of paths) {
if (!Object.keys(this.watchers).includes(path)) {
this.watchers[path] = await this.watcherInit(path)
console.log('added watcher', path)
@ -263,9 +272,8 @@ class FSPluginClient extends ElectronBasePluginClient {
const dirname = path.dirname(pathWithoutPrefix)
if (this.expandedPaths.includes(dirname) || this.expandedPaths.includes(pathWithoutPrefix)) {
console.log('emitting', eventName, pathWithoutPrefix, this.expandedPaths)
this.emit('change', eventName, pathWithoutPrefix)
this.dataBatcher.write('change', eventName, pathWithoutPrefix)
}
this.emit('change', eventName, pathWithoutPrefix)
} catch (e) {
console.log('error emitting change', e)
}
@ -276,7 +284,8 @@ class FSPluginClient extends ElectronBasePluginClient {
console.log('check emitting', eventName, pathWithoutPrefix, this.expandedPaths, dirname)
if (this.expandedPaths.includes(dirname) || this.expandedPaths.includes(pathWithoutPrefix)) {
console.log('emitting', eventName, pathWithoutPrefix, this.expandedPaths)
this.emit('change', eventName, pathWithoutPrefix)
//this.emit('change', eventName, pathWithoutPrefix)
this.dataBatcher.write('change', eventName, pathWithoutPrefix)
}
} catch (e) {
console.log('error emitting change', e)

@ -0,0 +1,62 @@
import {EventEmitter} from 'events';
import { StringDecoder } from 'string_decoder';
// Max duration to batch session data before sending it to the renderer process.
const BATCH_DURATION_MS = 16;
// Max size of a session data batch. Note that this value can be exceeded by ~4k
// (chunk sizes seem to be 4k at the most)
const BATCH_MAX_SIZE = 200 * 1024;
// Data coming from the pty is sent to the renderer process for further
// vt parsing and rendering. This class batches data to minimize the number of
// IPC calls. It also reduces GC pressure and CPU cost: each chunk is prefixed
// with the window ID which is then stripped on the renderer process and this
// overhead is reduced with batching.
export class PluginEventDataBatcher extends EventEmitter {
uid: number;
decoder: StringDecoder;
data!: any[]
timeout!: NodeJS.Timeout | null;
constructor(uid: number) {
super();
this.uid = uid;
this.decoder = new StringDecoder('utf8');
this.reset();
}
reset() {
this.data = [];
this.timeout = null;
}
write(key: string, ...payload: any): void {
if (this.data.length >= BATCH_MAX_SIZE) {
// We've reached the max batch size. Flush it and start another one
if (this.timeout) {
clearTimeout(this.timeout);
this.timeout = null;
}
this.flush();
}
this.data.push({
key,
payload,
});
console.log('data', this.data)
if (!this.timeout) {
this.timeout = setTimeout(() => this.flush(), BATCH_DURATION_MS);
}
}
flush() {
// Reset before emitting to allow for potential reentrancy
const data = this.data;
this.reset();
this.emit('flush', data);
}
}

@ -398,9 +398,9 @@ export const FileExplorer = (props: FileExplorerProps) => {
return (
<div ref={treeRef} tabIndex={0} style={{ outline: 'none',
display: 'flex',
flexDirection: 'column',
height: '100%',
display: 'flex',
flexDirection: 'column',
height: '100%',
}}>
<TreeView id="treeView">
<li key={`treeViewLiMenu`} data-id={`treeViewLiMenu`} className="li_tv">
@ -426,19 +426,19 @@ export const FileExplorer = (props: FileExplorerProps) => {
</div>
</li>
<div style={{flexGrow: 2}}>
<div>
<FlatTree
treeRef={treeRef}
handleTreeClick={handleTreeClick}
focusEdit={state.focusEdit}
focusElement={props.focusElement}
focusContext={state.focusContext}
editModeOff={editModeOff}
files={files}
expandPath={props.expandPath}
handleContextMenu={handleContextMenu}
/>
</div>
<div>
<FlatTree
treeRef={treeRef}
handleTreeClick={handleTreeClick}
focusEdit={state.focusEdit}
focusElement={props.focusElement}
focusContext={state.focusContext}
editModeOff={editModeOff}
files={files}
expandPath={props.expandPath}
handleContextMenu={handleContextMenu}
/>
</div>
</div>
<div className='d-block w-100 pb-4 mb-4'></div>

Loading…
Cancel
Save