Merge pull request #3996 from ethereum/circom-parser

Parse circom code and display errors and warnings in the editor

apps/circuit-compiler/project.json

@@ -0,0 +1,59 @@
+{
+    "name": "circuit-compiler",
+    "$schema": "../../node_modules/nx/schemas/project-schema.json",
+    "sourceRoot": "apps/circuit-compiler/src",
+    "projectType": "application",
+    "implicitDependencies": ["remixd"],
+    "targets": {
+      "build": {
+        "executor": "@nrwl/webpack:webpack",
+        "outputs": ["{options.outputPath}"],
+        "defaultConfiguration": "development",
+        "options": {
+          "compiler": "babel",
+          "outputPath": "dist/apps/circuit-compiler",
+          "index": "apps/circuit-compiler/src/index.html",
+          "baseHref": "./",
+          "main": "apps/circuit-compiler/src/main.tsx",
+          "polyfills": "apps/circuit-compiler/src/polyfills.ts",
+          "tsConfig": "apps/circuit-compiler/tsconfig.app.json",
+          "assets": ["apps/circuit-compiler/src/profile.json"],
+          "styles": ["apps/circuit-compiler/src/css/app.css"],
+          "scripts": [],
+          "webpackConfig": "apps/circuit-compiler/webpack.config.js"          
+        },
+        "configurations": {
+          "development": {
+          },
+          "production": {
+            "fileReplacements": [
+              {
+                "replace": "apps/circuit-compiler/src/environments/environment.ts",
+                "with": "apps/circuit-compiler/src/environments/environment.prod.ts"
+              }
+            ]
+          }
+        }
+      },
+      "serve": {
+        "executor": "@nrwl/webpack:dev-server",
+        "defaultConfiguration": "development",
+        "options": {
+          "buildTarget": "circuit-compiler:build",
+          "hmr": true,
+          "baseHref": "/"
+        },
+        "configurations": {
+          "development": {
+            "buildTarget": "circuit-compiler:build:development",
+            "port": 2023
+          },
+          "production": {
+            "buildTarget": "circuit-compiler:build:production"
+          }
+        }
+      }
+    },
+    "tags": []
+  }
+  

apps/circuit-compiler/src/app/app.tsx

@@ -0,0 +1,17 @@
+import React, { useEffect } from 'react'
+
+import { CircomPluginClient } from './services/circomPluginClient'
+
+function App() {
+
+  useEffect(() => {
+    new CircomPluginClient()
+  }, [])
+  
+  return (
+    <div className="App">
+    </div>
+  )
+}
+
+export default App

apps/circuit-compiler/src/app/services/circomPluginClient.ts

@@ -0,0 +1,224 @@
+import {PluginClient} from '@remixproject/plugin'
+import {createClient} from '@remixproject/plugin-webview'
+import EventManager from 'events'
+import pathModule from 'path'
+import {parse} from 'circom_wasm'
+
+export class CircomPluginClient extends PluginClient {
+  public internalEvents: EventManager
+
+  constructor() {
+    super()
+    createClient(this)
+    this.internalEvents = new EventManager()
+    this.methods = ['init', 'parse']
+    this.onload()
+  }
+
+  init(): void {
+    console.log('initializing circom plugin...')
+  }
+
+  onActivation(): void {
+    // @ts-ignore
+    this.on('editor', 'contentChanged', (path: string, fileContent) => {
+      if (path.endsWith('.circom')) {
+        this.parse(path, fileContent)
+      }
+    })
+  }
+
+  async parse(path: string, fileContent: string): Promise<void> {
+    let buildFiles = {
+      [path]: fileContent
+    }
+
+    buildFiles = await this.resolveDependencies(path, fileContent, buildFiles)
+    const parsedOutput = parse(path, buildFiles)
+
+    try {
+      const result = JSON.parse(parsedOutput)
+
+      if (result.length === 0) {
+        // @ts-ignore
+        await this.call('editor', 'clearErrorMarkers', [path])
+      } else {
+        const markers = []
+
+        for (const report of result) {
+          for (const label in report.labels) {
+            if (report.labels[label].file_id === '0') {
+              // @ts-ignore
+              const startPosition: {lineNumber: number; column: number} =
+                await this.call(
+                  'editor',
+                  // @ts-ignore
+                  'getPositionAt',
+                  report.labels[label].range.start
+                )
+              // @ts-ignore
+              const endPosition: {lineNumber: number; column: number} =
+                await this.call(
+                  'editor',
+                  // @ts-ignore
+                  'getPositionAt',
+                  report.labels[label].range.end
+                )
+
+              markers.push({
+                message: report.message,
+                severity: report.type.toLowerCase(),
+                position: {
+                  start: {
+                    line: startPosition.lineNumber,
+                    column: startPosition.column
+                  },
+                  end: {
+                    line: endPosition.lineNumber,
+                    column: endPosition.column
+                  }
+                },
+                file: path
+              })
+            }
+          }
+        }
+
+        if (markers.length > 0) {
+          // @ts-ignore
+          await this.call('editor', 'addErrorMarker', markers)
+        } else {
+          // @ts-ignore
+          await this.call('editor', 'clearErrorMarkers', [path])
+        }
+      }
+    } catch (e) {
+      console.log(e)
+    }
+  }
+
+  async resolveDependencies(
+    filePath: string,
+    fileContent: string,
+    output = {},
+    depPath: string = '',
+    blackPath: string[] = []
+  ): Promise<Record<string, string>> {
+    // extract all includes
+    const includes = (fileContent.match(/include ['"].*['"]/g) || []).map(
+      (include) => include.replace(/include ['"]/g, '').replace(/['"]/g, '')
+    )
+
+    await Promise.all(
+      includes.map(async (include) => {
+        // fix for endless recursive includes
+        if (blackPath.includes(include)) return
+        let dependencyContent = ''
+        let path = include
+        // @ts-ignore
+        const pathExists = await this.call('fileManager', 'exists', path)
+
+        if (pathExists) {
+          // fetch file content if include import (path) exists within same level as current file opened in editor
+          dependencyContent = await this.call('fileManager', 'readFile', path)
+        } else {
+          // if include import (path) does not exist, try to construct relative path using the original file path (current file opened in editor)
+          let relativePath = pathModule.resolve(
+            filePath.slice(0, filePath.lastIndexOf('/')),
+            include
+          )
+          if (relativePath.indexOf('/') === 0)
+            relativePath = relativePath.slice(1)
+          const relativePathExists = await this.call(
+            'fileManager',
+            // @ts-ignore
+            'exists',
+            relativePath
+          )
+
+          if (relativePathExists) {
+            // fetch file content if include import exists as a relative path
+            dependencyContent = await this.call(
+              'fileManager',
+              'readFile',
+              relativePath
+            )
+          } else {
+            if (depPath) {
+              // if depPath is provided, try to resolve include import from './deps' folder in remix
+              path = pathModule.resolve(
+                depPath.slice(0, depPath.lastIndexOf('/')),
+                include
+              )
+              if (path.indexOf('/') === 0) path = path.slice(1)
+              dependencyContent = await this.call(
+                'contentImport',
+                'resolveAndSave',
+                path,
+                null
+              )
+            } else {
+              if (include.startsWith('circomlib')) {
+                // try to resolve include import from github if it is a circomlib dependency
+                const splitInclude = include.split('/')
+                const version = splitInclude[1].match(/v[0-9]+.[0-9]+.[0-9]+/g)
+
+                if (version && version[0]) {
+                  path = `https://raw.githubusercontent.com/iden3/circomlib/${
+                    version[0]
+                  }/circuits/${splitInclude.slice(2).join('/')}`
+                  dependencyContent = await this.call(
+                    'contentImport',
+                    'resolveAndSave',
+                    path,
+                    null
+                  )
+                } else {
+                  path = `https://raw.githubusercontent.com/iden3/circomlib/master/circuits/${splitInclude
+                    .slice(1)
+                    .join('/')}`
+                  dependencyContent = await this.call(
+                    'contentImport',
+                    'resolveAndSave',
+                    path,
+                    null
+                  )
+                }
+              } else {
+                // If all import cases are not true, use the default import to try fetching from node_modules and unpkg
+                dependencyContent = await this.call(
+                  'contentImport',
+                  'resolveAndSave',
+                  path,
+                  null
+                )
+              }
+            }
+          }
+        }
+        // extract all includes from the dependency content
+        const dependencyIncludes = (
+          dependencyContent.match(/include ['"].*['"]/g) || []
+        ).map((include) =>
+          include.replace(/include ['"]/g, '').replace(/['"]/g, '')
+        )
+
+        blackPath.push(include)
+        // recursively resolve all dependencies of the dependency
+        if (dependencyIncludes.length > 0) {
+          await this.resolveDependencies(
+            filePath,
+            dependencyContent,
+            output,
+            path,
+            blackPath
+          )
+          output[include] = dependencyContent
+        } else {
+          output[include] = dependencyContent
+        }
+      })
+    )
+    return output
+  }
+}

apps/circuit-compiler/src/example/simple.circom

@@ -0,0 +1,11 @@
+pragma circom 2.0.0;
+
+template Multiplier2() {
+    signal input a;
+    signal input b;
+    signal output c;
+    c <== a*b;
+ }
+
+ component main = Multiplier2();
+ 

apps/circuit-compiler/src/index.html

@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <title>Circuit - Compiler</title>
+    <base href="./" />
+
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.8.1/css/all.css" integrity="sha384-50oBUHEmvpQ+1lW4y57PTFmhCaXp0ML5d60M1M7uH2+nqUivzIebhndOJK28anvf" crossorigin="anonymous"/>
+    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.2.3/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-rbsA2VBKQhggwzxH7pPCaAqO46MgnOM80zW1RWuH61DGLwZJEdK2Kadq2F9CUG65" crossorigin="anonymous">
+  </head>
+  <body>
+    <div id="root"></div>
+  </body>
+</html>

apps/circuit-compiler/src/main.tsx

@@ -0,0 +1,8 @@
+import React from 'react'
+import ReactDOM from 'react-dom'
+import App from './app/app'
+
+ReactDOM.render(
+  <App />,
+  document.getElementById('root')
+)

apps/circuit-compiler/src/polyfills.ts

@@ -0,0 +1,7 @@
+/**
+ * Polyfill stable language features. These imports will be optimized by `@babel/preset-env`.
+ *
+ * See: https://github.com/zloirock/core-js#babel
+ */
+import 'core-js/stable';
+import 'regenerator-runtime/runtime';

apps/circuit-compiler/src/profile.json

@@ -0,0 +1,17 @@
+{
+  "name": "circuit-compiler",
+  "kind": "provider",
+  "displayName": "Circuit Compiler",
+  "events": [],
+  "version": "2.0.0",
+  "methods": ["init", "parse"],
+  "canActivate": [],
+  "url": "",
+  "description": "Enables circuit compilation and computing a witness for ZK proofs",
+  "icon": "https://docs.circom.io/assets/images/favicon.png",
+  "location": "sidePanel",
+  "documentation": "",
+  "repo": "https://github.com/ethereum/remix-project/tree/master/apps/circuit-compiler",
+  "maintainedBy": "Remix",
+  "authorContact": ""
+}

apps/circuit-compiler/tsconfig.app.json

@@ -0,0 +1,24 @@
+{
+    "extends": "./tsconfig.json",
+    "compilerOptions": {
+      "outDir": "../../dist/out-tsc",
+      "types": ["node"]
+    },
+    "files": [
+      "../../node_modules/@nrwl/react/typings/cssmodule.d.ts",
+      "../../node_modules/@nrwl/react/typings/image.d.ts"
+    ],
+    "exclude": [
+      "jest.config.ts",
+      "**/*.spec.ts",
+      "**/*.test.ts",
+      "**/*.spec.tsx",
+      "**/*.test.tsx",
+      "**/*.spec.js",
+      "**/*.test.js",
+      "**/*.spec.jsx",
+      "**/*.test.jsx"
+    ],
+    "include": ["**/*.js", "**/*.jsx", "**/*.ts", "**/*.tsx"]
+  }
+  

apps/circuit-compiler/tsconfig.json

@@ -0,0 +1,17 @@
+{
+    "extends": "../../tsconfig.base.json",
+    "compilerOptions": {
+      "jsx": "react-jsx",
+      "allowJs": true,
+      "esModuleInterop": true,
+      "allowSyntheticDefaultImports": true
+    },
+    "files": [],
+    "include": [],
+    "references": [
+      {
+        "path": "./tsconfig.app.json"
+      }
+    ]
+  }
+  

apps/circuit-compiler/webpack.config.js

@@ -0,0 +1,92 @@
+const { composePlugins, withNx } = require('@nrwl/webpack')
+const webpack = require('webpack')
+const TerserPlugin = require("terser-webpack-plugin")
+const CssMinimizerPlugin = require("css-minimizer-webpack-plugin")
+
+// Nx plugins for webpack.
+module.exports = composePlugins(withNx(), (config) => {
+  // Update the webpack config as needed here.
+  // e.g. `config.plugins.push(new MyPlugin())`
+  // add fallback for node modules
+  config.resolve.fallback = {
+    ...config.resolve.fallback,
+    "crypto": require.resolve("crypto-browserify"),
+    "stream": require.resolve("stream-browserify"),
+    "path": require.resolve("path-browserify"),
+    "http": require.resolve("stream-http"),
+    "https": require.resolve("https-browserify"),
+    "constants": require.resolve("constants-browserify"),
+    "os": false, //require.resolve("os-browserify/browser"),
+    "timers": false, // require.resolve("timers-browserify"),
+    "zlib": require.resolve("browserify-zlib"),
+    "fs": false,
+    "module": false,
+    "tls": false,
+    "net": false,
+    "readline": false,
+    "child_process": false,
+    "buffer": require.resolve("buffer/"),
+    "vm": require.resolve('vm-browserify'),
+  }
+  
+
+  // add externals
+  config.externals = {
+    ...config.externals,
+    solc: 'solc',
+  }
+
+  // add public path
+  config.output.publicPath = './'
+
+  // add copy & provide plugin
+  config.plugins.push(
+    new webpack.ProvidePlugin({
+      Buffer: ['buffer', 'Buffer'],
+      url: ['url', 'URL'],
+      process: 'process/browser',
+    })
+  )
+
+  // set the define plugin to load the WALLET_CONNECT_PROJECT_ID
+  config.plugins.push(
+    new webpack.DefinePlugin({
+      WALLET_CONNECT_PROJECT_ID: JSON.stringify(process.env.WALLET_CONNECT_PROJECT_ID),
+    })
+  )
+
+  // souce-map loader
+  config.module.rules.push({
+    test: /\.js$/,
+    use: ["source-map-loader"],
+    enforce: "pre"
+  })
+
+  config.ignoreWarnings = [/Failed to parse source map/] // ignore source-map-loader warnings
+
+
+  // set minimizer
+  config.optimization.minimizer = [
+    new TerserPlugin({
+      parallel: true,
+      terserOptions: {
+        ecma: 2015,
+        compress: false,
+        mangle: false,
+        format: {
+          comments: false,
+        },
+      },
+      extractComments: false,
+    }),
+    new CssMinimizerPlugin(),
+  ];
+
+  config.watchOptions = {
+    ignored: /node_modules/
+  }
+
+  config.experiments.syncWebAssembly = true
+
+  return config;
+});

apps/remix-ide/project.json

@@ -3,7 +3,7 @@
   "$schema": "../../node_modules/nx/schemas/project-schema.json",
   "sourceRoot": "apps/remix-ide/src",
   "projectType": "application",
-  "implicitDependencies": ["doc-gen", "doc-viewer", "etherscan", "vyper", "solhint", "walletconnect"],
+  "implicitDependencies": ["doc-gen", "doc-viewer", "etherscan", "vyper", "solhint", "walletconnect", "circuit-compiler"],
   "targets": {
     "build": {
       "executor": "@nrwl/webpack:webpack",

apps/remix-ide/src/app.js

@@ -159,9 +159,10 @@ class AppComponent {
     // ----------------- editor service ----------------------------
     const editor = new Editor() // wrapper around ace editor
     Registry.getInstance().put({ api: editor, name: 'editor' })
-    editor.event.register('requiringToSaveCurrentfile', () =>
+    editor.event.register('requiringToSaveCurrentfile', (currentFile) => {
       fileManager.saveCurrentFile()
-    )
+      if (currentFile.endsWith('.circom')) this.appManager.activatePlugin(['circuit-compiler'])
+    })
 
     // ----------------- fileManager service ----------------------------
     const fileManager = new FileManager(editor, appManager)

apps/remix-ide/src/app/editor/editor.js

@@ -13,7 +13,7 @@ const profile = {
   name: 'editor',
   description: 'service - editor',
   version: packageJson.version,
-  methods: ['highlight', 'discardHighlight', 'clearAnnotations', 'addLineText', 'discardLineTexts', 'addAnnotation', 'gotoLine', 'revealRange', 'getCursorPosition', 'open', 'addModel','addErrorMarker', 'clearErrorMarkers', 'getText'],
+  methods: ['highlight', 'discardHighlight', 'clearAnnotations', 'addLineText', 'discardLineTexts', 'addAnnotation', 'gotoLine', 'revealRange', 'getCursorPosition', 'open', 'addModel','addErrorMarker', 'clearErrorMarkers', 'getText', 'getPositionAt'],
 }
 
 class Editor extends Plugin {
@@ -175,8 +175,8 @@ class Editor extends Plugin {
     }
 
     this.saveTimeout = window.setTimeout(() => {
-      this.triggerEvent('contentChanged', [])
+      this.triggerEvent('contentChanged', [currentFile, input])
-      this.triggerEvent('requiringToSaveCurrentfile', [])
+      this.triggerEvent('requiringToSaveCurrentfile', [currentFile])
     }, 500)
   }
 
@@ -579,6 +579,10 @@ class Editor extends Plugin {
       this.clearDecorationsByPlugin(session, from, 'lineTextPerFile', this.registeredDecorations, this.currentDecorations)
     }
   }
+
+  getPositionAt(offset) {
+    return this.api.getPositionAt(offset)
+  }
 }
 
 module.exports = Editor

apps/remix-ide/src/remixAppManager.js

@@ -17,7 +17,7 @@ const requiredModules = [ // services + layout views + system views
 // dependentModules shouldn't be manually activated (e.g hardhat is activated by remixd)
 const dependentModules = ['foundry', 'hardhat', 'truffle', 'slither']
 
-const loadLocalPlugins = ["doc-gen", "doc-viewer", "etherscan", "vyper", 'solhint', 'walletconnect']
+const loadLocalPlugins = ["doc-gen", "doc-viewer", "etherscan", "vyper", "solhint", "walletconnect", "circuit-compiler"]
 
 const sensitiveCalls = {
   'fileManager': ['writeFile', 'copyFile', 'rename', 'copyDir'],

apps/remix-ide/src/remixEngine.js

@@ -20,6 +20,7 @@ export class RemixEngine extends Engine {
     if (name === 'fetchAndCompile') return { queueTimeout: 60000 * 4 }
     if (name === 'walletconnect') return { queueTimeout: 60000 * 4 }
     if (name === 'udapp') return { queueTimeout: 60000 * 4 }
+    if (name === 'circuit-compiler') return { queueTimeout: 60000 * 4 }
     return { queueTimeout: 10000 }
   }
 

libs/remix-ui/editor/src/lib/remix-ui-editor.tsx

@@ -18,6 +18,7 @@ import {RemixDefinitionProvider} from './providers/definitionProvider'
 import {RemixCodeActionProvider} from './providers/codeActionProvider'
 import './remix-ui-editor.css'
 import {circomLanguageConfig, circomTokensProvider} from './syntaxes/circom'
+import {IPosition} from 'monaco-editor'
 
 enum MarkerSeverity {
   Hint = 1,
@@ -108,6 +109,7 @@ export type EditorAPIType = {
   keepDecorationsFor: (filePath: string, plugin: string, typeOfDecoration: string, registeredDecorations: any, currentDecorations: any) => DecorationsReturn
   addErrorMarker: (errors: errorMarker[], from: string) => void
   clearErrorMarkers: (sources: string[] | {[fileName: string]: any}, from: string) => void
+  getPositionAt: (offset: number) => monacoTypes.IPosition
 }
 
 /* eslint-disable-next-line */
@@ -550,6 +552,9 @@ export const EditorUI = (props: EditorUIProps) => {
     if (!editorRef.current) return
     return editorRef.current.getOption(43).fontSize
   }
+  props.editorAPI.getPositionAt = (offset: number): IPosition => {
+    return editorRef.current.getModel().getPositionAt(offset)
+  }
   ;(window as any).addRemixBreakpoint = (position) => {
     // make it available from e2e testing...
     const model = editorRef.current.getModel()

libs/remix-ui/helper/src/lib/helper-components.tsx

@@ -134,3 +134,11 @@ export const upgradeReportMsg = (report: LayoutCompatibilityReport) => (
     <div className="pl-4 text-danger">{report.explain()}</div>
   </div>
 )
+
+export function RenderIf({ condition, children }: { condition: boolean, children: JSX.Element }) {
+  return condition ? children : null
+}
+
+export function RenderIfNot({ condition, children }: { condition: boolean, children: JSX.Element }) {
+  return condition ? null : children
+}

libs/remix-ui/solidity-compiler/src/lib/compiler-container.tsx

@@ -515,7 +515,6 @@ export const CompilerContainer = (props: CompilerContainerProps) => {
     const currentFile = api.currentFile
 
     if (!isSolFileSelected()) return
-
     _setCompilerVersionFromPragma(currentFile)
     let externalCompType
     if (hhCompilation) externalCompType = 'hardhat'
@@ -527,7 +526,6 @@ export const CompilerContainer = (props: CompilerContainerProps) => {
     const currentFile = api.currentFile
 
     if (!isSolFileSelected()) return
-
     _setCompilerVersionFromPragma(currentFile)
     let externalCompType
     if (hhCompilation) externalCompType = 'hardhat'

libs/remix-ui/tabs/src/lib/remix-ui-tabs.tsx

@@ -167,7 +167,7 @@ export const TabsUI = (props: TabsUIProps) => {
           <button
             data-id="play-editor"
             className="btn text-success py-0"
-            disabled={!(tabsState.currentExt === 'js' || tabsState.currentExt === 'ts' || tabsState.currentExt === 'sol')}
+            disabled={!(tabsState.currentExt === 'js' || tabsState.currentExt === 'ts' || tabsState.currentExt === 'sol' || tabsState.currentExt === 'circom')}
             onClick={async () => {
               const path = active().substr(active().indexOf('/') + 1, active().length)
               const content = await props.plugin.call('fileManager', 'readFile', path)

package.json

@@ -150,6 +150,7 @@
     "brace": "^0.8.0",
     "change-case": "^4.1.1",
     "chokidar": "^2.1.8",
+    "circom_wasm": "^0.0.2",
     "color-support": "^1.1.3",
     "commander": "^9.4.1",
     "core-js": "^3.6.5",

yarn.lock

@@ -9897,6 +9897,11 @@ cipher-base@^1.0.0, cipher-base@^1.0.1, cipher-base@^1.0.3:
     inherits "^2.0.1"
     safe-buffer "^5.0.1"
 
+circom_wasm@^0.0.2:
+  version "0.0.2"
+  resolved "https://registry.yarnpkg.com/circom_wasm/-/circom_wasm-0.0.2.tgz#9d24866b8289a5778999270823a4cb06e64145b5"
+  integrity sha512-SCMP6cxHHL7MLedDrTl+nGYyE6+kE5GepbxtZm65GlR0wUMD9eNOD1shwScWaDnmBOZTrImmNeTYZA5DWCmIww==
+
 circular-json@^0.3.0:
   version "0.3.3"
   resolved "https://registry.yarnpkg.com/circular-json/-/circular-json-0.3.3.tgz#815c99ea84f6809529d2f45791bdf82711352d66"