Merge pull request #1216 from rocky/ast-functions

Add routines for finding AST nodes

.gitignore

@@ -14,3 +14,5 @@ package-lock.json
 TODO
 soljson.js
 lerna-debug.log
+*~
+/tmp

remix-astwalker/src/astWalker.ts

@@ -4,6 +4,21 @@ import { AstNodeLegacy, Node, AstNode } from "./index";
 export declare interface AstWalker {
   new(): EventEmitter;
 }
+
+const isObject = function(obj: any): boolean {
+  return obj != null && obj.constructor.name === "Object"
+}
+
+export function isAstNode(node: Object): boolean {
+  return (
+    isObject(node) &&
+    'id' in node &&
+    'nodeType' in node &&
+    'src' in node
+  )
+}
+
+
 /**
  * Crawl the given AST through the function walk(ast, callback)
  */
@@ -22,6 +37,10 @@ export class AstWalker extends EventEmitter {
     node: AstNodeLegacy | AstNode,
     callback: Object | Function
   ): any {
+    // FIXME: we shouldn't be doing this callback determination type on each AST node,
+    // since the callback function is set once per walk.
+    // Better would be to store the right one as a variable and
+    // return that.
     if (<AstNodeLegacy>node) {
       if ((<AstNodeLegacy>node).name in callback) {
         return callback[(<AstNodeLegacy>node).name](node);
@@ -98,6 +117,35 @@ export class AstWalker extends EventEmitter {
     }
   }
 
+  walkFullInternal(ast: AstNode, callback: Function) {
+
+    if (isAstNode(ast)) {
+      // console.log(`XXX id ${ast.id}, nodeType: ${ast.nodeType}, src: ${ast.src}`);
+      callback(ast);
+      for (let k of Object.keys(ast)) {
+        // Possible optimization:
+        // if (k in ['id', 'src', 'nodeType']) continue;
+        const astItem = ast[k];
+        if (Array.isArray(astItem)) {
+          for (let child of astItem) {
+            if (child) {
+              this.walkFullInternal(child, callback);
+            }
+          }
+        } else {
+          this.walkFullInternal(astItem, callback);
+        }
+      }
+    }
+  }
+
+  // Normalizes parameter callback and calls walkFullInternal
+  walkFull(ast: AstNode, callback: any) {
+    if (!isAstNode(ast)) throw new TypeError("first argument should be an ast");
+    return this.walkFullInternal(ast, callback);
+  }
+
+
   walkAstList(sourcesList: Node, cb?: Function) {
     if (cb) {
       if (sourcesList.ast) {

remix-astwalker/src/index.ts

@@ -1,2 +1,3 @@
 export * from './types'
 export * from './astWalker'
+export * from './sourceMappings'

remix-astwalker/src/sourceMappings.ts

@@ -0,0 +1,93 @@
+import { isAstNode, AstWalker } from './astWalker';
+import { AstNode, Location } from "./types";
+
+export declare interface SourceMappings {
+  new(): SourceMappings;
+}
+
+/**
+ * Break out fields of an AST's "src" attribute string (s:l:f)
+ * into its "start", "length", and "file index" components.
+ *
+ * @param {AstNode} astNode  - the object to convert.
+ */
+export function sourceLocationFromAstNode(astNode: AstNode): Location | null {
+  if (isAstNode(astNode) && astNode.src) {
+    var split = astNode.src.split(':')
+    return <Location>{
+      start: parseInt(split[0], 10),
+      length: parseInt(split[1], 10),
+      file: parseInt(split[2], 10)
+    }
+  }
+  return null;
+}
+
+/**
+ * Routines for retrieving AST object(s) using some criteria, usually
+ * includng "src' information.
+ */
+export class SourceMappings {
+
+  readonly source: string;
+  readonly lineBreaks: Array<number>;
+
+  constructor(source: string) {
+    this.source = source;
+
+    // Create a list of line offsets which will be used to map between
+    // character offset and line/column positions.
+    let lineBreaks: Array<number> = [];
+    for (var pos = source.indexOf('\n'); pos >= 0; pos = source.indexOf('\n', pos + 1)) {
+      lineBreaks.push(pos)
+    }
+    this.lineBreaks = lineBreaks;
+  };
+
+  /**
+   * get a list of nodes that are at the given @arg position
+   *
+   * @param {String} astNodeType - type of node to return or null
+   * @param {Int} position     - character offset
+   * @return {Object} ast object given by the compiler
+   */
+  nodesAtPosition(astNodeType: string | null, position: Location, ast: AstNode): Array<AstNode> {
+    const astWalker = new AstWalker()
+    let found: Array<AstNode> = [];
+
+    const callback = function(node: AstNode): boolean {
+      let nodeLocation = sourceLocationFromAstNode(node);
+      if (nodeLocation &&
+        nodeLocation.start == position.start &&
+        nodeLocation.length == position.length) {
+        if (!astNodeType || astNodeType === node.nodeType) {
+          found.push(node)
+        }
+      }
+      return true;
+    }
+    astWalker.walkFull(ast, callback);
+    return found;
+  }
+
+  findNodeAtSourceLocation(astNodeType: string | undefined, sourceLocation: Location, ast: AstNode | null): AstNode | null {
+    const astWalker = new AstWalker()
+    let found = null;
+    /* FIXME: Looking at AST walker code,
+       I don't understand a need to return a boolean. */
+    const callback = function(node: AstNode) {
+      let nodeLocation = sourceLocationFromAstNode(node);
+      if (nodeLocation &&
+        nodeLocation.start == sourceLocation.start &&
+        nodeLocation.length == sourceLocation.length) {
+        if (astNodeType == undefined || astNodeType === node.nodeType) {
+          found = node;
+        }
+      }
+      return true;
+    }
+
+    astWalker.walkFull(ast, callback);
+    return found;
+  }
+}

remix-astwalker/src/types.ts

@@ -1,3 +1,9 @@
+export interface Location {
+  start: number;
+  length: number;
+  file: number;  // Would it be clearer to call this a file index?
+}
+
 export interface Node {
   ast?: AstNode;
   legacyAST?: AstNodeLegacy;
@@ -6,12 +12,15 @@ export interface Node {
 }
 
 export interface AstNode {
+  /* The following fields are essential, and indicates an that object
+     is an AST node. */
+  id: number;  // This is unique across all nodes in an AST tree
+  nodeType: string;
+  src: string;
+
   absolutePath?: string;
   exportedSymbols?: Object;
-  id: number;
-  nodeType: string;
   nodes?: Array<AstNode>;
-  src: string;
   literals?: Array<string>;
   file?: string;
   scope?: number;
@@ -21,10 +30,10 @@ export interface AstNode {
 }
 
 export interface AstNodeLegacy {
-  id: number;
+  id: number;    // This is unique across all nodes in an AST tree
-  name: string;
+  name: string;  // This corresponds to "nodeType" in ASTNode
   src: string;
-  children?: Array<AstNodeLegacy>;
+  children?: Array<AstNodeLegacy>;  // This corresponds to "nodes" in ASTNode
   attributes?: AstNodeAtt;
 }
 

remix-astwalker/tests/newTests.ts

@@ -1,12 +1,13 @@
 import tape from "tape";
-import { AstWalker, AstNode } from "../src";
+import { AstWalker, AstNode, isAstNode } from "../src";
 import node from "./resources/newAST";
+import legacyNode from "./resources/legacyAST";
 
 tape("New ASTWalker", (t: tape.Test) => {
-  t.test("ASTWalker.walk && .walkAST", (st: tape.Test) => {
-    st.plan(24);
   // New Ast Object
   const astWalker = new AstWalker();
+  t.test("ASTWalker.walk && .walkastList", (st: tape.Test) => {
+    st.plan(24);
     // EventListener
     astWalker.on("node", node => {
       if (node.nodeType === "ContractDefinition") {
@@ -51,6 +52,30 @@ tape("New ASTWalker", (t: tape.Test) => {
     });
     st.end();
   });
+  t.test("ASTWalkFull", (st: tape.Test) => {
+    const astNodeCount = 26;
+    st.plan(2 + astNodeCount);
+    let count: number = 0;
+    astWalker.walkFull(node.ast, (node: AstNode) => {
+      st.ok(isAstNode(node), "passed an ast node");
+      count += 1;
+    });
+    st.equal(count, astNodeCount, "traverses all AST nodes");
+    count = 0;
+    let badCall = function() {
+      /* Typescript will keep us from calling walkFull with a legacyAST.
+	 However, for non-typescript uses, we add this test which casts
+	 to an AST to check that there is a run-time check in walkFull.
+      */
+      astWalker.walkFull(<AstNode>legacyNode, (node: AstNode) => {
+        count += 1;
+      });
+    }
+    t.throws(badCall, /first argument should be an ast/,
+      "passing legacyAST fails");
+    st.equal(count, 0, "traverses no AST nodes");
+    st.end();
+  });
 });
 
 function checkProgramDirective(st: tape.Test, node: AstNode) {

remix-astwalker/tests/resources/newAST.ts

@@ -1,7 +1,297 @@
 import { Node } from '../../src/'
 let node: Node;
 
-node = { "ast": { "absolutePath": "greeter.sol", "exportedSymbols": { "Greeter": [25] }, "id": 26, "nodeType": "SourceUnit", "nodes": [{ "id": 1, "literals": ["solidity", ">=", "0.5", ".0", "<", "0.6", ".0"], "nodeType": "PragmaDirective", "src": "0:31:0" }, { "absolutePath": "mortal.sol", "file": "mortal.sol", "id": 2, "nodeType": "ImportDirective", "scope": 26, "sourceUnit": 53, "src": "32:20:0", "symbolAliases": [], "unitAlias": "" }, { "baseContracts": [{ "arguments": null, "baseName": { "contractScope": null, "id": 3, "name": "Mortal", "nodeType": "UserDefinedTypeName", "referencedDeclaration": 52, "src": "74:6:0", "typeDescriptions": { "typeIdentifier": "t_contract$_Mortal_$52", "typeString": "contract Mortal" } }, "id": 4, "nodeType": "InheritanceSpecifier", "src": "74:6:0" }], "contractDependencies": [52], "contractKind": "contract", "documentation": null, "fullyImplemented": true, "id": 25, "linearizedBaseContracts": [25, 52], "name": "Greeter", "nodeType": "ContractDefinition", "nodes": [{ "constant": false, "id": 6, "name": "greeting", "nodeType": "VariableDeclaration", "scope": 25, "src": "141:15:0", "stateVariable": true, "storageLocation": "default", "typeDescriptions": { "typeIdentifier": "t_string_storage", "typeString": "string" }, "typeName": { "id": 5, "name": "string", "nodeType": "ElementaryTypeName", "src": "141:6:0", "typeDescriptions": { "typeIdentifier": "t_string_storage_ptr", "typeString": "string" } }, "value": null, "visibility": "internal" }, { "body": { "id": 15, "nodeType": "Block", "src": "257:37:0", "statements": [{ "expression": { "argumentTypes": null, "id": 13, "isConstant": false, "isLValue": false, "isPure": false, "lValueRequested": false, "leftHandSide": { "argumentTypes": null, "id": 11, "name": "greeting", "nodeType": "Identifier", "overloadedDeclarations": [], "referencedDeclaration": 6, "src": "267:8:0", "typeDescriptions": { "typeIdentifier": "t_string_storage", "typeString": "string storage ref" } }, "nodeType": "Assignment", "operator": "=", "rightHandSide": { "argumentTypes": null, "id": 12, "name": "_greeting", "nodeType": "Identifier", "overloadedDeclarations": [], "referencedDeclaration": 8, "src": "278:9:0", "typeDescriptions": { "typeIdentifier": "t_string_memory_ptr", "typeString": "string memory" } }, "src": "267:20:0", "typeDescriptions": { "typeIdentifier": "t_string_storage", "typeString": "string storage ref" } }, "id": 14, "nodeType": "ExpressionStatement", "src": "267:20:0" }] }, "documentation": null, "id": 16, "implemented": true, "kind": "constructor", "modifiers": [], "name": "", "nodeType": "FunctionDefinition", "parameters": { "id": 9, "nodeType": "ParameterList", "parameters": [{ "constant": false, "id": 8, "name": "_greeting", "nodeType": "VariableDeclaration", "scope": 16, "src": "225:23:0", "stateVariable": false, "storageLocation": "memory", "typeDescriptions": { "typeIdentifier": "t_string_memory_ptr", "typeString": "string" }, "typeName": { "id": 7, "name": "string", "nodeType": "ElementaryTypeName", "src": "225:6:0", "typeDescriptions": { "typeIdentifier": "t_string_storage_ptr", "typeString": "string" } }, "value": null, "visibility": "internal" }], "src": "224:25:0" }, "returnParameters": { "id": 10, "nodeType": "ParameterList", "parameters": [], "src": "257:0:0" }, "scope": 25, "src": "213:81:0", "stateMutability": "nonpayable", "superFunction": null, "visibility": "public" }, { "body": { "id": 23, "nodeType": "Block", "src": "377:32:0", "statements": [{ "expression": { "argumentTypes": null, "id": 21, "name": "greeting", "nodeType": "Identifier", "overloadedDeclarations": [], "referencedDeclaration": 6, "src": "394:8:0", "typeDescriptions": { "typeIdentifier": "t_string_storage", "typeString": "string storage ref" } }, "functionReturnParameters": 20, "id": 22, "nodeType": "Return", "src": "387:15:0" }] }, "documentation": null, "id": 24, "implemented": true, "kind": "function", "modifiers": [], "name": "greet", "nodeType": "FunctionDefinition", "parameters": { "id": 17, "nodeType": "ParameterList", "parameters": [], "src": "338:2:0" }, "returnParameters": { "id": 20, "nodeType": "ParameterList", "parameters": [{ "constant": false, "id": 19, "name": "", "nodeType": "VariableDeclaration", "scope": 24, "src": "362:13:0", "stateVariable": false, "storageLocation": "memory", "typeDescriptions": { "typeIdentifier": "t_string_memory_ptr", "typeString": "string" }, "typeName": { "id": 18, "name": "string", "nodeType": "ElementaryTypeName", "src": "362:6:0", "typeDescriptions": { "typeIdentifier": "t_string_storage_ptr", "typeString": "string" } }, "value": null, "visibility": "internal" }], "src": "361:15:0" }, "scope": 25, "src": "324:85:0", "stateMutability": "view", "superFunction": null, "visibility": "public" }], "scope": 26, "src": "54:357:0" }], "src": "0:412:0" }, "id": 0 }
+node = {
+  "ast":
+  {
+    "absolutePath": "greeter.sol",
+    "exportedSymbols": {
+      "Greeter": [
+        25
+      ]
+    },
+    "id": 26,
+    "nodeType": "SourceUnit",
+    "nodes": [
+      {
+        "id": 1,
+        "literals": [
+          "solidity",
+          ">=",
+          "0.5",
+          ".0",
+          "<",
+          "0.6",
+          ".0"
+        ],
+        "nodeType": "PragmaDirective",
+        "src": "0:31:0"
+      },
+      {
+        "absolutePath": "mortal.sol",
+        "file": "mortal.sol",
+        "id": 2,
+        "nodeType": "ImportDirective",
+        "scope": 26,
+        "sourceUnit": 53,
+        "src": "32:20:0",
+        "symbolAliases": [],
+        "unitAlias": ""
+      },
+      {
+        "baseContracts": [
+          {
+            "arguments": null,
+            "baseName": {
+              "contractScope": null,
+              "id": 3,
+              "name": "Mortal",
+              "nodeType": "UserDefinedTypeName",
+              "referencedDeclaration": 52,
+              "src": "74:6:0",
+              "typeDescriptions": {
+                "typeIdentifier": "t_contract$_Mortal_$52",
+                "typeString": "contract Mortal"
+              }
+            },
+            "id": 4,
+            "nodeType": "InheritanceSpecifier",
+            "src": "74:6:0"
+          }
+        ],
+        "contractDependencies": [
+          52
+        ],
+        "contractKind": "contract",
+        "documentation": null,
+        "fullyImplemented": true,
+        "id": 25,
+        "linearizedBaseContracts": [
+          25,
+          52
+        ],
+        "name": "Greeter",
+        "nodeType": "ContractDefinition",
+        "nodes": [
+          {
+            "constant": false,
+            "id": 6,
+            "name": "greeting",
+            "nodeType": "VariableDeclaration",
+            "scope": 25,
+            "src": "141:15:0",
+            "stateVariable": true,
+            "storageLocation": "default",
+            "typeDescriptions": {
+              "typeIdentifier": "t_string_storage",
+              "typeString": "string"
+            },
+            "typeName": {
+              "id": 5,
+              "name": "string",
+              "nodeType": "ElementaryTypeName",
+              "src": "141:6:0",
+              "typeDescriptions": {
+                "typeIdentifier": "t_string_storage_ptr",
+                "typeString": "string"
+              }
+            },
+            "value": null,
+            "visibility": "internal"
+          },
+          {
+            "body": {
+              "id": 15,
+              "nodeType": "Block",
+              "src": "257:37:0",
+              "statements": [
+                {
+                  "expression": {
+                    "argumentTypes": null,
+                    "id": 13,
+                    "isConstant": false,
+                    "isLValue": false,
+                    "isPure": false,
+                    "lValueRequested": false,
+                    "leftHandSide": {
+                      "argumentTypes": null,
+                      "id": 11,
+                      "name": "greeting",
+                      "nodeType": "Identifier",
+                      "overloadedDeclarations": [],
+                      "referencedDeclaration": 6,
+                      "src": "267:8:0",
+                      "typeDescriptions": {
+                        "typeIdentifier": "t_string_storage",
+                        "typeString": "string storage ref"
+                      }
+                    },
+                    "nodeType": "Assignment",
+                    "operator": "=",
+                    "rightHandSide": {
+                      "argumentTypes": null,
+                      "id": 12,
+                      "name": "_greeting",
+                      "nodeType": "Identifier",
+                      "overloadedDeclarations": [],
+                      "referencedDeclaration": 8,
+                      "src": "278:9:0",
+                      "typeDescriptions": {
+                        "typeIdentifier": "t_string_memory_ptr",
+                        "typeString": "string memory"
+                      }
+                    },
+                    "src": "267:20:0",
+                    "typeDescriptions": {
+                      "typeIdentifier": "t_string_storage",
+                      "typeString": "string storage ref"
+                    }
+                  },
+                  "id": 14,
+                  "nodeType": "ExpressionStatement",
+                  "src": "267:20:0"
+                }
+              ]
+            },
+            "documentation": null,
+            "id": 16,
+            "implemented": true,
+            "kind": "constructor",
+            "modifiers": [],
+            "name": "",
+            "nodeType": "FunctionDefinition",
+            "parameters": {
+              "id": 9,
+              "nodeType": "ParameterList",
+              "parameters": [
+                {
+                  "constant": false,
+                  "id": 8,
+                  "name": "_greeting",
+                  "nodeType": "VariableDeclaration",
+                  "scope": 16,
+                  "src": "225:23:0",
+                  "stateVariable": false,
+                  "storageLocation": "memory",
+                  "typeDescriptions": {
+                    "typeIdentifier": "t_string_memory_ptr",
+                    "typeString": "string"
+                  },
+                  "typeName": {
+                    "id": 7,
+                    "name": "string",
+                    "nodeType": "ElementaryTypeName",
+                    "src": "225:6:0",
+                    "typeDescriptions": {
+                      "typeIdentifier": "t_string_storage_ptr",
+                      "typeString": "string"
+                    }
+                  },
+                  "value": null,
+                  "visibility": "internal"
+                }
+              ],
+              "src": "224:25:0"
+            },
+            "returnParameters": {
+              "id": 10,
+              "nodeType": "ParameterList",
+              "parameters": [],
+              "src": "257:0:0"
+            },
+            "scope": 25,
+            "src": "213:81:0",
+            "stateMutability": "nonpayable",
+            "superFunction": null,
+            "visibility": "public"
+          },
+          {
+            "body": {
+              "id": 23,
+              "nodeType": "Block",
+              "src": "377:32:0",
+              "statements": [
+                {
+                  "expression": {
+                    "argumentTypes": null,
+                    "id": 21,
+                    "name": "greeting",
+                    "nodeType": "Identifier",
+                    "overloadedDeclarations": [],
+                    "referencedDeclaration": 6,
+                    "src": "394:8:0",
+                    "typeDescriptions": {
+                      "typeIdentifier": "t_string_storage",
+                      "typeString": "string storage ref"
+                    }
+                  },
+                  "functionReturnParameters": 20,
+                  "id": 22,
+                  "nodeType": "Return",
+                  "src": "387:15:0"
+                }
+              ]
+            },
+            "documentation": null,
+            "id": 24,
+            "implemented": true,
+            "kind": "function",
+            "modifiers": [],
+            "name": "greet",
+            "nodeType": "FunctionDefinition",
+            "parameters": {
+              "id": 17,
+              "nodeType": "ParameterList",
+              "parameters": [],
+              "src": "338:2:0"
+            },
+            "returnParameters": {
+              "id": 20,
+              "nodeType": "ParameterList",
+              "parameters": [
+                {
+                  "constant": false,
+                  "id": 19,
+                  "name": "",
+                  "nodeType": "VariableDeclaration",
+                  "scope": 24,
+                  "src": "362:13:0",
+                  "stateVariable": false,
+                  "storageLocation": "memory",
+                  "typeDescriptions": {
+                    "typeIdentifier": "t_string_memory_ptr",
+                    "typeString": "string"
+                  },
+                  "typeName": {
+                    "id": 18,
+                    "name": "string",
+                    "nodeType": "ElementaryTypeName",
+                    "src": "362:6:0",
+                    "typeDescriptions": {
+                      "typeIdentifier": "t_string_storage_ptr",
+                      "typeString": "string"
+                    }
+                  },
+                  "value": null,
+                  "visibility": "internal"
+                }
+              ],
+              "src": "361:15:0"
+            },
+            "scope": 25,
+            "src": "324:85:0",
+            "stateMutability": "view",
+            "superFunction": null,
+            "visibility": "public"
+          }
+        ],
+        "scope": 26,
+        "src": "54:357:0"
+      }
+    ],
+    "src": "0:412:0"
+  }
+}
 
 
 node.source = `contract test {

remix-astwalker/tests/sourceMappings.ts

@@ -0,0 +1,44 @@
+import tape from "tape";
+import { AstNode, isAstNode, SourceMappings, sourceLocationFromAstNode } from "../src";
+import node from "./resources/newAST";
+
+tape("SourceMappings", (t: tape.Test) => {
+  const source = node.source;
+  const srcMappings = new SourceMappings(source);
+  t.test("SourceMappings constructor", (st: tape.Test) => {
+    st.plan(2)
+    st.equal(srcMappings.source, source, "sourceMappings object has source-code string");
+    st.deepEqual(srcMappings.lineBreaks,
+      [15, 26, 27, 38, 39, 81, 87, 103, 119, 135, 141, 142, 186, 192, 193, 199],
+      "sourceMappings has line-break offsets");
+    st.end();
+  });
+  t.test("SourceMappings functions", (st: tape.Test) => {
+    // st.plan(2)
+    const ast = node.ast;
+    st.deepEqual(sourceLocationFromAstNode(ast.nodes[0]),
+      { start: 0, length: 31, file: 0 },
+      "sourceLocationFromAstNode extracts a location");
+
+    /* Typescript will keep us from calling sourceLocationFromAstNode
+       with the wrong type. However, for non-typescript uses, we add
+       this test which casts to an AST to check that there is a
+       run-time check in walkFull.
+    */
+    st.notOk(sourceLocationFromAstNode(<AstNode>null),
+      "sourceLocationFromAstNode rejects an invalid astNode");
+    const loc = { start: 267, length: 20, file: 0 };
+    let astNode = srcMappings.findNodeAtSourceLocation('ExpressionStatement', loc, ast);
+    st.ok(isAstNode(astNode), "findsNodeAtSourceLocation finds something");
+    astNode = srcMappings.findNodeAtSourceLocation('NotARealThingToFind', loc, ast);
+    st.notOk(isAstNode(astNode),
+      "findsNodeAtSourceLocation fails to find something when it should");
+    let astNodes = srcMappings.nodesAtPosition(null, loc, ast);
+    st.equal(astNodes.length, 2, "nodesAtPosition should find more than one astNode");
+    st.ok(isAstNode(astNodes[0]), "nodesAtPosition returns only AST nodes");
+    // console.log(astNodes[0]);
+    astNodes = srcMappings.nodesAtPosition("ExpressionStatement", loc, ast);
+    st.equal(astNodes.length, 1, "nodesAtPosition filtered to a single nodeType");
+    st.end();
+  });
+});