Merge pull request #2908 from ethereum/parserfix

Parserfix
pull/2933/head
bunsenstraat 2 years ago committed by GitHub
commit 9dc5d09d4d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 4
      apps/remix-ide/src/app/editor/editor.js
  2. 20
      apps/remix-ide/src/app/plugins/parser/code-parser.tsx
  3. 54
      apps/remix-ide/src/app/plugins/parser/services/antlr-worker.ts
  4. 172
      apps/remix-ide/src/app/plugins/parser/services/code-parser-antlr-service.ts
  5. 4
      apps/remix-ide/src/app/plugins/parser/services/code-parser-compiler.ts
  6. 1948
      apps/remix-ide/src/assets/js/parser/antlr.js
  7. 6
      apps/remix-ide/src/assets/js/parser/antlr.js.map
  8. 130
      libs/remix-ui/editor/src/lib/providers/completionProvider.ts
  9. 6
      libs/remix-ui/editor/src/lib/remix-ui-editor.tsx

@ -358,8 +358,8 @@ class Editor extends Plugin {
/**
* The position of the cursor
*/
getCursorPosition () {
return this.api.getCursorPosition()
getCursorPosition (offset = true) {
return this.api.getCursorPosition(offset)
}
/**

@ -58,8 +58,6 @@ interface codeParserIndex {
export class CodeParser extends Plugin {
antlrParserResult: antlr.ParseResult // contains the simple parsed AST for the current file
compilerAbstract: CompilerAbstract
currentFile: string
nodeIndex: codeParserIndex
@ -77,7 +75,7 @@ export class CodeParser extends Plugin {
getLastNodeInLine: (ast: string) => Promise<any>
listAstNodes: () => Promise<any>
getANTLRBlockAtPosition: (position: any, text?: string) => Promise<any>
getCurrentFileAST: (text?: string) => Promise<ParseResult>
setCurrentFileAST: (text?: string) => Promise<ParseResult>
getImports: () => Promise<CodeParserImportsData[]>
@ -94,7 +92,9 @@ export class CodeParser extends Plugin {
async handleChangeEvents() {
const completionSettings = await this.call('config', 'getAppParameter', 'auto-completion')
if (completionSettings) {
await this.antlrService.getCurrentFileAST()
this.antlrService.enableWorker()
} else {
this.antlrService.disableWorker()
}
const showGasSettings = await this.call('config', 'getAppParameter', 'show-gas')
const showErrorSettings = await this.call('config', 'getAppParameter', 'display-errors')
@ -114,7 +114,7 @@ export class CodeParser extends Plugin {
this.getLastNodeInLine = this.antlrService.getLastNodeInLine.bind(this.antlrService)
this.listAstNodes = this.antlrService.listAstNodes.bind(this.antlrService)
this.getANTLRBlockAtPosition = this.antlrService.getANTLRBlockAtPosition.bind(this.antlrService)
this.getCurrentFileAST = this.antlrService.getCurrentFileAST.bind(this.antlrService)
this.setCurrentFileAST = this.antlrService.setCurrentFileAST.bind(this.antlrService)
this.getImports = this.importService.getImports.bind(this.importService)
this.on('editor', 'didChangeFile', async (file) => {
@ -134,10 +134,12 @@ export class CodeParser extends Plugin {
await this.importService.setFileTree()
})
this.on('fileManager', 'currentFileChanged', async () => {
await this.call('editor', 'discardLineTexts')
const completionSettings = await this.call('config', 'getAppParameter', 'auto-completion')
if (completionSettings) {
this.antlrService.setCurrentFileAST()
}
await this.handleChangeEvents()
})
@ -413,6 +415,7 @@ export class CodeParser extends Plugin {
return nodeDefinition
} else {
const astNodes = await this.antlrService.listAstNodes()
if (astNodes && astNodes.length) {
for (const node of astNodes) {
if (node.range[0] <= position && node.range[1] >= position) {
if (nodeDefinition && nodeDefinition.range[0] < node.range[0]) {
@ -427,6 +430,7 @@ export class CodeParser extends Plugin {
}
return nodeDefinition
}
}
}
@ -559,7 +563,7 @@ export class CodeParser extends Plugin {
async getNodeLink(node: genericASTNode) {
const lineColumn = await this.getLineColumnOfNode(node)
const position = await this.positionOfDefinition(node)
if (this.compilerAbstract && this.compilerAbstract.source) {
if (this.compilerAbstract && this.compilerAbstract.source && position) {
const fileName = this.compilerAbstract.getSourceName(position.file)
return lineColumn ? `${fileName} ${lineColumn.start.line}:${lineColumn.start.column}` : null
}

@ -0,0 +1,54 @@
let parser: any
// 'DedicatedWorkerGlobalScope' object (the Worker global scope) is accessible through the self keyword
// 'dom' and 'webworker' library files can't be included together https://github.com/microsoft/TypeScript/issues/20595
export default function (self) { // eslint-disable-line @typescript-eslint/explicit-module-boundary-types
self.addEventListener('message', (e) => {
const data: any = e.data
switch (data.cmd) {
case 'load':
{
delete self.Module
// NOTE: workaround some browsers?
self.Module = undefined
// importScripts() method of synchronously imports one or more scripts into the worker's scope
self.importScripts(e.data.url)
// @ts-ignore
parser = SolidityParser as any;
self.postMessage({
cmd: 'loaded',
})
break
}
case 'parse':
if (data.text && parser) {
try {
let startTime = performance.now()
const blocks = parser.parseBlock(data.text, { loc: true, range: true, tolerant: true })
const blockDuration = performance.now() - startTime
startTime = performance.now()
const ast = parser.parse(data.text, { loc: true, range: true, tolerant: true })
const endTime = performance.now()
self.postMessage({
cmd: 'parsed',
timestamp: data.timestamp,
ast,
text: data.text,
file: data.file,
duration: endTime - startTime,
blockDuration,
blocks
})
} catch (e) {
// do nothing
}
}
break
}
}, false)
}

@ -3,18 +3,109 @@
import { AstNode } from "@remix-project/remix-solidity-ts"
import { CodeParser } from "../code-parser"
import { antlr } from '../types'
import work from 'webworkify-webpack'
const SolidityParser = (window as any).SolidityParser = (window as any).SolidityParser || []
interface BlockDefinition {
end: number
endColumn: number
endLine: number
name: string
parent: string
start: number
startColumn: number
startLine: number
type: string
}
export default class CodeParserAntlrService {
plugin: CodeParser
worker: Worker
parserStartTime: number = 0
workerTimer: NodeJS.Timer
parserTreshHold: number = 10
cache: {
[name: string]: {
text: string,
ast: antlr.ParseResult | null,
duration?: number,
blockDuration?: number,
parsingEnabled?: boolean,
blocks?: BlockDefinition[]
}
} = {};
constructor(plugin: CodeParser) {
this.plugin = plugin
this.createWorker()
}
/*
* simple parsing is used to quickly parse the current file or a text source without using the compiler or having to resolve imports
*/
createWorker() {
this.worker = work(require.resolve('./antlr-worker'));
this.worker.postMessage({
cmd: 'load',
url: document.location.protocol + '//' + document.location.host + '/assets/js/parser/antlr.js',
});
const self = this
this.worker.addEventListener('message', function (ev) {
switch (ev.data.cmd) {
case 'parsed':
if (ev.data.ast && self.parserStartTime === ev.data.timestamp) {
self.setFileParsingState(ev.data.file, ev.data.blockDuration)
self.cache[ev.data.file] = {
...self.cache[ev.data.file],
text: ev.data.text,
ast: ev.data.ast,
duration: ev.data.duration,
blockDuration: ev.data.blockDuration,
blocks: ev.data.blocks,
}
}
break;
}
});
}
setFileParsingState(file: string, duration: number) {
if (this.cache[file]) {
if (this.cache[file].blockDuration) {
if (this.cache[file].blockDuration > this.parserTreshHold && duration > this.parserTreshHold) {
this.cache[file].parsingEnabled = false
this.plugin.call('notification', 'toast', `This file is big so some autocomplete features will be disabled.`)
} else {
this.cache[file].parsingEnabled = true
}
}
}
}
enableWorker() {
if (!this.workerTimer) {
this.workerTimer = setInterval(() => {
this.setCurrentFileAST()
}, 5000)
}
}
disableWorker() {
clearInterval(this.workerTimer)
}
async parseWithWorker(text: string, file: string) {
this.parserStartTime = Date.now()
this.worker.postMessage({
cmd: 'parse',
text,
timestamp: this.parserStartTime,
file,
parsingEnabled: (this.cache[file] && this.cache[file].parsingEnabled) || true
});
}
async parseSolidity(text: string) {
const ast: antlr.ParseResult = (SolidityParser as any).parse(text, { loc: true, range: true, tolerant: true })
@ -27,18 +118,28 @@ export default class CodeParserAntlrService {
* @param text
* @returns
*/
async getCurrentFileAST(text: string | null = null) {
async setCurrentFileAST(text: string | null = null) {
try {
this.plugin.currentFile = await this.plugin.call('fileManager', 'file')
if (this.plugin.currentFile && this.plugin.currentFile.endsWith('.sol')) {
if (!this.plugin.currentFile) return
const fileContent = text || await this.plugin.call('fileManager', 'readFile', this.plugin.currentFile)
if (!this.cache[this.plugin.currentFile]) {
this.cache[this.plugin.currentFile] = {
text: '',
ast: null,
parsingEnabled: true
}
}
if (this.cache[this.plugin.currentFile] && this.cache[this.plugin.currentFile].text !== fileContent) {
try {
const ast = await this.parseSolidity(fileContent)
this.plugin.antlrParserResult = ast
await this.parseWithWorker(fileContent, this.plugin.currentFile)
} catch (e) {
// do nothing
}
return this.plugin.antlrParserResult
}
}
} catch (e) {
// do nothing
}
}
@ -48,9 +149,10 @@ export default class CodeParserAntlrService {
* @returns
*/
async listAstNodes() {
await this.getCurrentFileAST();
this.plugin.currentFile = await this.plugin.call('fileManager', 'file')
if (!this.cache[this.plugin.currentFile]) return
const nodes: AstNode[] = [];
(SolidityParser as any).visit(this.plugin.antlrParserResult, {
(SolidityParser as any).visit(this.cache[this.plugin.currentFile].ast, {
StateVariableDeclaration: (node: antlr.StateVariableDeclaration) => {
if (node.variables) {
for (const variable of node.variables) {
@ -127,6 +229,29 @@ export default class CodeParserAntlrService {
}
return lastNode
}
/*
* get the code blocks of the current file
*/
async getCurrentFileBlocks(text: string | null = null) {
this.plugin.currentFile = await this.plugin.call('fileManager', 'file')
if (this.cache[this.plugin.currentFile]) {
if (!this.cache[this.plugin.currentFile].parsingEnabled) {
return
}
}
if (this.plugin.currentFile && this.plugin.currentFile.endsWith('.sol')) {
const fileContent = text || await this.plugin.call('fileManager', 'readFile', this.plugin.currentFile)
try {
const startTime = Date.now()
const blocks = (SolidityParser as any).parseBlock(fileContent, { loc: true, range: true, tolerant: true })
this.setFileParsingState(this.plugin.currentFile, Date.now() - startTime)
if (blocks) this.cache[this.plugin.currentFile].blocks = blocks
return blocks
} catch (e) {
// do nothing
}
}
}
/**
* Returns the block surrounding the given position
@ -136,23 +261,22 @@ export default class CodeParserAntlrService {
* @return {any}
* */
async getANTLRBlockAtPosition(position: any, text: string = null) {
await this.getCurrentFileAST(text)
const allowedTypes = ['SourceUnit', 'ContractDefinition', 'FunctionDefinition']
const walkAst = (node: any) => {
if (node.loc.start.line <= position.lineNumber && node.loc.end.line >= position.lineNumber) {
const children = node.children || node.subNodes
if (children && allowedTypes.indexOf(node.type) !== -1) {
for (const child of children) {
const result = walkAst(child)
if (result) return result
}
const blocks: any[] = await this.getCurrentFileBlocks(text)
const walkAst = (blocks) => {
let nodeFound = null
for (const object of blocks) {
if (object.start <= position) {
nodeFound = object
break
}
return node
}
return null
return nodeFound
}
if (!this.plugin.antlrParserResult) return
return walkAst(this.plugin.antlrParserResult)
if (!blocks) return
blocks.reverse()
const block = walkAst(blocks)
return block
}
}

@ -134,8 +134,8 @@ export default class CodeParserCompiler {
"language": "Solidity",
"settings": {
"optimizer": {
"enabled": false,
"runs": 200
"enabled": state.optimize,
"runs": state.runs
},
"outputSelection": {
"*": {

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

@ -1,4 +1,3 @@
import { sourceMappingDecoder } from "@remix-project/remix-debug"
import { AstNode } from "@remix-project/remix-solidity-ts"
import { isArray } from "lodash"
import { editor, languages, Position } from "monaco-editor"
@ -10,6 +9,7 @@ export class RemixCompletionProvider implements languages.CompletionItemProvider
props: EditorUIProps
monaco: any
maximumItemsForContractCompletion = 100
constructor(props: any, monaco: any) {
this.props = props
@ -21,6 +21,7 @@ export class RemixCompletionProvider implements languages.CompletionItemProvider
const completionSettings = await this.props.plugin.call('config', 'getAppParameter', 'settings/auto-completion')
if (!completionSettings) return
const word = model.getWordUntilPosition(position);
const range = {
startLineNumber: position.lineNumber,
@ -32,12 +33,9 @@ export class RemixCompletionProvider implements languages.CompletionItemProvider
const line = model.getLineContent(position.lineNumber)
let nodes: AstNode[] = []
let suggestions: monaco.languages.CompletionItem[] = []
if (context.triggerCharacter === '"' || context.triggerCharacter === '@' || context.triggerCharacter === '/') {
const lastpart = line.substring(0, position.column - 1).split(';').pop()
if (lastpart.startsWith('import')) {
const imports = await this.props.plugin.call('codeParser', 'getImports')
if (context.triggerCharacter === '"' || context.triggerCharacter === '@') {
@ -53,9 +51,8 @@ export class RemixCompletionProvider implements languages.CompletionItemProvider
return
}
}
} else
if (context.triggerCharacter === '.') {
} else if (context.triggerCharacter === '.') {
const lineTextBeforeCursor: string = line.substring(0, position.column - 1)
const lastNodeInExpression = await this.getLastNodeInExpression(lineTextBeforeCursor)
const expressionElements = lineTextBeforeCursor.split('.')
@ -70,21 +67,17 @@ export class RemixCompletionProvider implements languages.CompletionItemProvider
if (globalCompletion) {
dotCompleted = true
suggestions = [...suggestions, ...globalCompletion]
setTimeout(() => {
// eslint-disable-next-line no-debugger
// debugger
}, 2000)
}
// handle completion for global THIS.
if (lastNodeInExpression.name === 'this') {
dotCompleted = true
nodes = [...nodes, ...await this.getThisCompletions(position)]
nodes = [...nodes, ...await this.getThisCompletions()]
}
// handle completion for other dot completions
if (expressionElements.length > 1 && !dotCompleted) {
const nameOfLastTypedExpression = lastNodeInExpression.name || lastNodeInExpression.memberName
const dotCompletions = await this.getDotCompletions(position, nameOfLastTypedExpression, range)
const dotCompletions = await this.getDotCompletions(nameOfLastTypedExpression, range)
nodes = [...nodes, ...dotCompletions.nodes]
suggestions = [...suggestions, ...dotCompletions.suggestions]
}
@ -99,7 +92,8 @@ export class RemixCompletionProvider implements languages.CompletionItemProvider
...GetGlobalFunctions(range, this.monaco),
...GeCompletionUnits(range, this.monaco),
]
let contractCompletions = await this.getContractCompletions(position)
let contractCompletions = await this.getContractCompletions()
// we can't have external nodes without using this.
contractCompletions = contractCompletions.filter(node => {
@ -108,13 +102,14 @@ export class RemixCompletionProvider implements languages.CompletionItemProvider
}
return true
})
nodes = [...nodes, ...contractCompletions]
}
// remove duplicates
const nodeIds = {};
const filteredNodes = nodes.filter((node) => {
let filteredNodes = nodes.filter((node) => {
if (node.id) {
if (nodeIds[node.id]) {
return false;
@ -124,6 +119,12 @@ export class RemixCompletionProvider implements languages.CompletionItemProvider
return true;
});
// truncate for performance
if (filteredNodes.length > this.maximumItemsForContractCompletion) {
await this.props.plugin.call('notification', 'toast', `Too many completion items. Only ${this.maximumItemsForContractCompletion} items will be shown.`)
filteredNodes = filteredNodes.slice(0, this.maximumItemsForContractCompletion)
}
const getNodeLink = async (node: any) => {
return await this.props.plugin.call('codeParser', 'getNodeLink', node)
}
@ -250,73 +251,16 @@ export class RemixCompletionProvider implements languages.CompletionItemProvider
}
}
private getBlockNodesAtPosition = async (position: Position) => {
let nodes: any[] = []
const cursorPosition = this.props.editorAPI.getCursorPosition()
const nodesAtPosition = await this.props.plugin.call('codeParser', 'nodesAtPosition', cursorPosition)
// try to get the block from ANTLR of which the position is in
const ANTLRBlock = await this.props.plugin.call('codeParser', 'getANTLRBlockAtPosition', position, null)
// if the block has a name and a type we can maybe find it in the contract nodes
const fileNodes = await this.props.plugin.call('codeParser', 'getCurrentFileNodes')
if (isArray(nodesAtPosition) && nodesAtPosition.length) {
for (const node of nodesAtPosition) {
// try to find the real block in the AST and get the nodes in that scope
if (node.nodeType === 'ContractDefinition') {
const contractNodes = fileNodes.contracts[node.name].contractNodes
for (const contractNode of Object.values(contractNodes)) {
if (contractNode['name'] === ANTLRBlock.name
|| (contractNode['kind'] === 'constructor' && ANTLRBlock.name === null)
) {
let nodeOfScope = await this.props.plugin.call('codeParser', 'getNodesWithScope', (contractNode as any).id)
nodes = [...nodes, ...nodeOfScope]
if (contractNode['body']) {
nodeOfScope = await this.props.plugin.call('codeParser', 'getNodesWithScope', (contractNode['body'] as any).id)
nodes = [...nodes, ...nodeOfScope]
}
}
}
}
// blocks can have statements
/*
if (node.statements){
console.log('statements', node.statements)
for (const statement of node.statements) {
if(statement.expression){
const declaration = await this.props.plugin.call('codeParser', 'declarationOf', statement.expression)
declaration.outSideBlock = true
nodes = [...nodes, declaration]
}
}
}
*/
}
}
// we are only interested in nodes that are in the same block as the cursor
nodes = nodes.filter(node => {
if (node.src) {
const position = sourceMappingDecoder.decode(node.src)
if (position.start >= ANTLRBlock.range[0] && (position.start + position.length) <= ANTLRBlock.range[1]) {
return true
}
}
if (node.outSideBlock) { return true }
return false
})
return nodes;
}
private getContractCompletions = async (position: Position) => {
private getContractCompletions = async () => {
let nodes: any[] = []
const cursorPosition = this.props.editorAPI.getCursorPosition()
let nodesAtPosition = await this.props.plugin.call('codeParser', 'nodesAtPosition', cursorPosition)
// if no nodes exits at position, try to get the block of which the position is in
const block = await this.props.plugin.call('codeParser', 'getANTLRBlockAtPosition', position, null)
const block = await this.props.plugin.call('codeParser', 'getANTLRBlockAtPosition', cursorPosition, null)
const fileNodes = await this.props.plugin.call('codeParser', 'getCurrentFileNodes')
if (!nodesAtPosition.length) {
if (block) {
nodesAtPosition = await this.props.plugin.call('codeParser', 'nodesAtPosition', block.range[0])
nodesAtPosition = await this.props.plugin.call('codeParser', 'nodesAtPosition', block.start)
}
}
// find the contract and get the nodes of the contract and the base contracts and imports
@ -325,13 +269,26 @@ export class RemixCompletionProvider implements languages.CompletionItemProvider
for (const node of nodesAtPosition) {
if (node.nodeType === 'ContractDefinition') {
contractNode = node
const fileNodes = await this.props.plugin.call('codeParser', 'getCurrentFileNodes')
const contractNodes = fileNodes.contracts[node.name]
nodes = [...Object.values(contractNodes.contractScopeNodes), ...nodes]
nodes = [...Object.values(contractNodes.baseNodesWithBaseContractScope), ...nodes]
nodes = [...Object.values(fileNodes.imports), ...nodes]
// at the nodes at the block itself
nodes = [...nodes, ...await this.getBlockNodesAtPosition(position)]
// add the nodes at the block itself
if (node.nodeType === 'ContractDefinition' && block && block.name) {
const contractNodes = fileNodes.contracts[node.name].contractNodes
for (const contractNode of Object.values(contractNodes)) {
if (contractNode['name'] === block.name
|| (contractNode['kind'] === 'constructor' && block.name === 'constructor')
) {
let nodeOfScope = await this.props.plugin.call('codeParser', 'getNodesWithScope', (contractNode as any).id)
nodes = [...nodes, ...nodeOfScope]
if (contractNode['body']) {
nodeOfScope = await this.props.plugin.call('codeParser', 'getNodesWithScope', (contractNode['body'] as any).id)
nodes = [...nodes, ...nodeOfScope]
}
}
}
}
// filter private nodes, only allow them when contract ID is the same as the current contract
nodes = nodes.filter(node => {
if (node.visibility) {
@ -347,14 +304,17 @@ export class RemixCompletionProvider implements languages.CompletionItemProvider
}
} else {
// get all the nodes from a simple code parser which only parses the current file
nodes = [...nodes, ...await this.props.plugin.call('codeParser', 'listAstNodes')]
const allNodesFromAntlr = await this.props.plugin.call('codeParser', 'listAstNodes')
if (allNodesFromAntlr) {
nodes = [...nodes, ...allNodesFromAntlr]
}
}
return nodes
}
private getThisCompletions = async (position: Position) => {
private getThisCompletions = async () => {
let nodes: any[] = []
let thisCompletionNodes = await this.getContractCompletions(position)
let thisCompletionNodes = await this.getContractCompletions()
const allowedTypesForThisCompletion = ['VariableDeclaration', 'FunctionDefinition']
// with this. you can't have internal nodes and no contractDefinitions
thisCompletionNodes = thisCompletionNodes.filter(node => {
@ -367,15 +327,11 @@ export class RemixCompletionProvider implements languages.CompletionItemProvider
return true
})
nodes = [...nodes, ...thisCompletionNodes]
setTimeout(() => {
// eslint-disable-next-line no-debugger
// debugger
}, 2000)
return nodes
}
private getDotCompletions = async (position: Position, nameOfLastTypedExpression: string, range) => {
const contractCompletions = await this.getContractCompletions(position)
private getDotCompletions = async (nameOfLastTypedExpression: string, range) => {
const contractCompletions = await this.getContractCompletions()
let nodes: any[] = []
let suggestions: monaco.languages.CompletionItem[] = []

@ -103,7 +103,7 @@ export interface EditorUIProps {
findMatches: (uri: string, value: string) => any
getFontSize: () => number,
getValue: (uri: string) => string
getCursorPosition: () => number
getCursorPosition: (offset?: boolean) => number | IPosition
getHoverPosition: (position: IPosition) => number
addDecoration: (marker: sourceMarker, filePath: string, typeOfDecoration: string) => DecorationsReturn
clearDecorationsByPlugin: (filePath: string, plugin: string, typeOfDecoration: string, registeredDecorations: any, currentDecorations: any) => DecorationsReturn
@ -480,11 +480,11 @@ export const EditorUI = (props: EditorUIProps) => {
}
}
props.editorAPI.getCursorPosition = () => {
props.editorAPI.getCursorPosition = (offset:boolean = true) => {
if (!monacoRef.current) return
const model = editorModelsState[currentFileRef.current]?.model
if (model) {
return model.getOffsetAt(editorRef.current.getPosition())
return offset? model.getOffsetAt(editorRef.current.getPosition()): editorRef.current.getPosition()
}
}

Loading…
Cancel
Save