Merge pull request #1528 from ethereum/github

github integration/dgit github support

apps/remix-ide/src/app/files/dgitProvider.js

@@ -8,6 +8,7 @@ import IpfsHttpClient from 'ipfs-http-client'
 import {
   saveAs
 } from 'file-saver'
+import http from 'isomorphic-git/http/web'
 
 const JSZip = require('jszip')
 const path = require('path')
@@ -20,7 +21,7 @@ const profile = {
   description: '',
   icon: 'assets/img/fileManager.webp',
   version: '0.0.1',
-  methods: ['init', 'status', 'log', 'commit', 'add', 'remove', 'rm', 'lsfiles', 'readblob', 'resolveref', 'branches', 'branch', 'checkout', 'currentbranch', 'push', 'pin', 'pull', 'pinList', 'unPin', 'setIpfsConfig', 'zip', 'setItem', 'getItem'],
+  methods: ['init', 'localStorageUsed', 'addremote', 'delremote', 'remotes', 'fetch', 'clone', 'export', 'import', 'status', 'log', 'commit', 'add', 'remove', 'rm', 'lsfiles', 'readblob', 'resolveref', 'branches', 'branch', 'checkout', 'currentbranch', 'push', 'pin', 'pull', 'pinList', 'unPin', 'setIpfsConfig', 'zip', 'setItem', 'getItem'],
   kind: 'file-system'
 }
 class DGitProvider extends Plugin {
@@ -55,10 +56,24 @@ class DGitProvider extends Plugin {
     }
   }
 
-  async init () {
+  async parseInput (input) {
+    return {
+      corsProxy: 'https://corsproxy.remixproject.org/',
+      http,
+      onAuth: url => {
+        const auth = {
+          username: input.token,
+          password: ''
+        }
+        return auth
+      }
+    }
+  }
+
+  async init (input) {
     await git.init({
       ...await this.getGitConfig(),
-      defaultBranch: 'main'
+      defaultBranch: (input && input.branch) || 'main'
     })
   }
 
@@ -75,7 +90,7 @@ class DGitProvider extends Plugin {
       ...await this.getGitConfig(),
       ...cmd
     })
-    this.call('fileManager', 'refresh')
+    await this.call('fileManager', 'refresh')
   }
 
   async rm (cmd) {
@@ -83,7 +98,7 @@ class DGitProvider extends Plugin {
       ...await this.getGitConfig(),
       ...cmd
     })
-    this.call('fileManager', 'refresh')
+    await this.call('fileManager', 'refresh')
   }
 
   async checkout (cmd) {
@@ -91,7 +106,7 @@ class DGitProvider extends Plugin {
       ...await this.getGitConfig(),
       ...cmd
     })
-    this.call('fileManager', 'refresh')
+    await this.call('fileManager', 'refresh')
   }
 
   async log (cmd) {
@@ -102,12 +117,21 @@ class DGitProvider extends Plugin {
     return status
   }
 
+  async remotes () {
+    let remotes = []
+    try {
+      remotes = await git.listRemotes({ ...await this.getGitConfig() })
+    } catch (e) {
+    }
+    return remotes
+  }
+
   async branch (cmd) {
     const status = await git.branch({
       ...await this.getGitConfig(),
       ...cmd
     })
-    this.call('fileManager', 'refresh')
+    await this.call('fileManager', 'refresh')
     return status
   }
 
@@ -119,9 +143,17 @@ class DGitProvider extends Plugin {
   }
 
   async branches () {
-    const branches = await git.listBranches({
+    const cmd = {
       ...await this.getGitConfig()
-    })
+    }
+    const remotes = await this.remotes()
+    let branches = []
+    branches = (await git.listBranches(cmd)).map((branch) => { return { remote: undefined, name: branch } })
+    for (const remote of remotes) {
+      cmd.remote = remote.remote
+      const remotebranches = (await git.listBranches(cmd)).map((branch) => { return { remote: remote.remote, name: branch } })
+      branches = [...branches, ...remotebranches]
+    }
     return branches
   }
 
@@ -133,7 +165,9 @@ class DGitProvider extends Plugin {
         ...cmd
       })
       return sha
-    } catch (e) {}
+    } catch (e) {
+      throw new Error(e)
+    }
   }
 
   async lsfiles (cmd) {
@@ -177,8 +211,90 @@ class DGitProvider extends Plugin {
     }
   }
 
-  async push () {
+  async addremote (input) {
-    if (!this.checkIpfsConfig()) return false
+    await git.addRemote({ ...await this.getGitConfig(), url: input.url, remote: input.remote })
+  }
+
+  async delremote (input) {
+    await git.deleteRemote({ ...await this.getGitConfig(), remote: input.remote })
+  }
+
+  async localStorageUsed () {
+    return this.calculateLocalStorage()
+  }
+
+  async clone (input) {
+    const permission = await this.askUserPermission('clone', 'Import multiple files into your workspaces.')
+    if (!permission) return false
+    if (this.calculateLocalStorage() > 10000) throw new Error('The local storage of the browser is full.')
+    await this.call('filePanel', 'createWorkspace', `workspace_${Date.now()}`, false)
+
+    const cmd = {
+      url: input.url,
+      singleBranch: input.singleBranch,
+      ref: input.branch,
+      depth: input.depth || 10,
+      ...await this.parseInput(input),
+      ...await this.getGitConfig()
+    }
+
+    const result = await git.clone(cmd)
+    await this.call('fileManager', 'refresh')
+    return result
+  }
+
+  async push (input) {
+    const cmd = {
+      force: input.force,
+      ref: input.ref,
+      remoteRef: input.remoteRef,
+      remote: input.remote,
+      author: {
+        name: input.name,
+        email: input.email
+      },
+      ...await this.parseInput(input),
+      ...await this.getGitConfig()
+    }
+    return await git.push(cmd)
+  }
+
+  async pull (input) {
+    const cmd = {
+      ref: input.ref,
+      remoteRef: input.remoteRef,
+      author: {
+        name: input.name,
+        email: input.email
+      },
+      remote: input.remote,
+      ...await this.parseInput(input),
+      ...await this.getGitConfig()
+    }
+    const result = await git.pull(cmd)
+    await this.call('fileManager', 'refresh')
+    return result
+  }
+
+  async fetch (input) {
+    const cmd = {
+      ref: input.ref,
+      remoteRef: input.remoteRef,
+      author: {
+        name: input.name,
+        email: input.email
+      },
+      remote: input.remote,
+      ...await this.parseInput(input),
+      ...await this.getGitConfig()
+    }
+    const result = await git.fetch(cmd)
+    await this.call('fileManager', 'refresh')
+    return result
+  }
+
+  async export (config) {
+    if (!this.checkIpfsConfig(config)) return false
     const workspace = await this.call('filePanel', 'getCurrentWorkspace')
     const files = await this.getDirectory('/')
     this.filesToSend = []
@@ -255,7 +371,8 @@ class DGitProvider extends Plugin {
             pinata_secret_api_key: pinataSecretApiKey
           }
         })
-      return result.data.IpfsHash
+      // also commit to remix IPFS for availability after pinning to Pinata
+      return await this.export(this.remixIPFS) || result.data.IpfsHash
     } catch (error) {
       throw new Error(error)
     }
@@ -312,12 +429,13 @@ class DGitProvider extends Plugin {
         const dir = path.dirname(file.path)
         try {
           this.createDirectories(`${workspace.absolutePath}/${dir}`)
-        } catch (e) {}
+        } catch (e) { throw new Error(e) }
         try {
           window.remixFileSystem.writeFileSync(`${workspace.absolutePath}/${file.path}`, Buffer.concat(content) || new Uint8Array())
-        } catch (e) {}
+        } catch (e) { throw new Error(e) }
       }
     } catch (e) {
+      throw new Error(e)
     }
     return result
   }
@@ -336,14 +454,19 @@ class DGitProvider extends Plugin {
     return (_lsTotal / 1024).toFixed(2)
   }
 
-  async pull (cmd) {
+  async import (cmd) {
-    const permission = await this.askUserPermission('pull', 'Import multiple files into your workspaces.')
+    const permission = await this.askUserPermission('import', 'Import multiple files into your workspaces.')
     if (!permission) return false
-    if (this.calculateLocalStorage() > 10000) throw new Error('Local browser storage is full.')
+    if (this.calculateLocalStorage() > 10000) throw new Error('The local storage of the browser is full.')
     const cid = cmd.cid
     await this.call('filePanel', 'createWorkspace', `workspace_${Date.now()}`, false)
     const workspace = await this.call('filePanel', 'getCurrentWorkspace')
-    const result = await this.importIPFSFiles(this.remixIPFS, cid, workspace) || await this.importIPFSFiles(this.ipfsconfig, cid, workspace) || await this.importIPFSFiles(this.globalIPFSConfig, cid, workspace)
+    let result
+    if (cmd.local) {
+      result = await this.importIPFSFiles(this.ipfsconfig, cid, workspace)
+    } else {
+      result = await this.importIPFSFiles(this.remixIPFS, cid, workspace) || await this.importIPFSFiles(this.ipfsconfig, cid, workspace) || await this.importIPFSFiles(this.globalIPFSConfig, cid, workspace)
+    }
     await this.call('fileManager', 'refresh')
     if (!result) throw new Error(`Cannot pull files from IPFS at ${cid}`)
   }
@@ -359,7 +482,8 @@ class DGitProvider extends Plugin {
       if (typeof window !== 'undefined') {
         window.localStorage.setItem(name, content)
       }
-    } catch (exception) {
+    } catch (e) {
+      console.log(e)
       return false
     }
     return true
@@ -391,8 +515,12 @@ class DGitProvider extends Plugin {
       if (i > 0) previouspath = '/' + directories.slice(0, i).join('/')
       const finalPath = previouspath + '/' + directories[i]
       try {
+        if (!window.remixFileSystem.existsSync(finalPath)) {
           window.remixFileSystem.mkdirSync(finalPath)
-      } catch (e) {}
+        }
+      } catch (e) {
+        console.log(e)
+      }
     }
   }
 

apps/remix-ide/src/app/files/fileManager.js

@@ -22,7 +22,7 @@ const profile = {
   icon: 'assets/img/fileManager.webp',
   permission: true,
   version: packageJson.version,
-  methods: ['file', 'exists', 'open', 'writeFile', 'readFile', 'copyFile', 'copyDir', 'rename', 'mkdir', 'readdir', 'remove', 'getCurrentFile', 'getFile', 'getFolder', 'setFile', 'switchFile', 'refresh', 'getProviderOf', 'getProviderByName', 'getPathFromUrl', 'getUrlFromPath', 'saveCurrentFile'],
+  methods: ['closeAllFiles', 'closeFile', 'file', 'exists', 'open', 'writeFile', 'readFile', 'copyFile', 'copyDir', 'rename', 'mkdir', 'readdir', 'remove', 'getCurrentFile', 'getFile', 'getFolder', 'setFile', 'switchFile', 'refresh', 'getProviderOf', 'getProviderByName', 'getPathFromUrl', 'getUrlFromPath', 'saveCurrentFile'],
   kind: 'file-system'
 }
 const errorMsg = {
@@ -61,6 +61,10 @@ class FileManager extends Plugin {
     return path.replace(/^\/browser\//, '').replace(/^browser\//, '') // forbids plugin to access the root filesystem
   }
 
+  normalize (path) {
+    return path.replace(/^\/+/, '')
+  }
+
   /**
    * Emit error if path doesn't exist
    * @param {string} path path of the file/directory
@@ -119,6 +123,7 @@ class FileManager extends Plugin {
    */
   exists (path) {
     try {
+      path = this.normalize(path)
       path = this.limitPluginScope(path)
       const provider = this.fileProviderOf(path)
       const result = provider.exists(path)
@@ -134,8 +139,8 @@ class FileManager extends Plugin {
   */
   refresh () {
     const provider = this.fileProviderOf('/')
-    // emit folderAdded so that File Explorer reloads the file tree
+    // emit rootFolderChanged so that File Explorer reloads the file tree
-    provider.event.emit('folderAdded', '/')
+    provider.event.emit('rootFolderChanged')
   }
 
   /**
@@ -168,6 +173,7 @@ class FileManager extends Plugin {
    * @returns {void}
    */
   async open (path) {
+    path = this.normalize(path)
     path = this.limitPluginScope(path)
     path = this.getPathFromUrl(path).file
     await this._handleExists(path, `Cannot open file ${path}`)
@@ -183,6 +189,7 @@ class FileManager extends Plugin {
    */
   async writeFile (path, data) {
     try {
+      path = this.normalize(path)
       path = this.limitPluginScope(path)
       if (await this.exists(path)) {
         await this._handleIsFile(path, `Cannot write file ${path}`)
@@ -204,6 +211,7 @@ class FileManager extends Plugin {
    */
   async readFile (path) {
     try {
+      path = this.normalize(path)
       path = this.limitPluginScope(path)
       await this._handleExists(path, `Cannot read file ${path}`)
       await this._handleIsFile(path, `Cannot read file ${path}`)
@@ -221,6 +229,8 @@ class FileManager extends Plugin {
    */
   async copyFile (src, dest, customName) {
     try {
+      src = this.normalize(src)
+      dest = this.normalize(dest)
       src = this.limitPluginScope(src)
       dest = this.limitPluginScope(dest)
       await this._handleExists(src, `Cannot copy from ${src}. Path does not exist.`)
@@ -245,6 +255,8 @@ class FileManager extends Plugin {
    */
   async copyDir (src, dest) {
     try {
+      src = this.normalize(src)
+      dest = this.normalize(dest)
       src = this.limitPluginScope(src)
       dest = this.limitPluginScope(dest)
       await this._handleExists(src, `Cannot copy from ${src}. Path does not exist.`)
@@ -281,6 +293,8 @@ class FileManager extends Plugin {
    */
   async rename (oldPath, newPath) {
     try {
+      oldPath = this.normalize(oldPath)
+      newPath = this.normalize(newPath)
       oldPath = this.limitPluginScope(oldPath)
       newPath = this.limitPluginScope(newPath)
       await this._handleExists(oldPath, `Cannot rename ${oldPath}`)
@@ -313,6 +327,7 @@ class FileManager extends Plugin {
    */
   async mkdir (path) {
     try {
+      path = this.normalize(path)
       path = this.limitPluginScope(path)
       if (await this.exists(path)) {
         throw createError({ code: 'EEXIST', message: `Cannot create directory ${path}` })
@@ -332,6 +347,7 @@ class FileManager extends Plugin {
    */
   async readdir (path) {
     try {
+      path = this.normalize(path)
       path = this.limitPluginScope(path)
       await this._handleExists(path)
       await this._handleIsDir(path)
@@ -356,6 +372,7 @@ class FileManager extends Plugin {
    */
   async remove (path) {
     try {
+      path = this.normalize(path)
       path = this.limitPluginScope(path)
       await this._handleExists(path, `Cannot remove file or directory ${path}`)
       const provider = this.fileProviderOf(path)
@@ -508,6 +525,7 @@ class FileManager extends Plugin {
   }
 
   _setFileInternal (path, content) {
+    console.log(path, content)
     const provider = this.fileProviderOf(path)
     if (!provider) throw createError({ code: 'ENOENT', message: `${path} not available` })
     // TODO : Add permission
@@ -708,7 +726,9 @@ class FileManager extends Plugin {
   }
 
   syncEditor (path) {
+    console.log(path)
     var currentFile = this._deps.config.get('currentFile')
+    console.log(currentFile, path)
     if (path !== currentFile) return
 
     var provider = this.fileProviderOf(currentFile)

apps/remix-ide/src/app/files/fileProvider.js

@@ -276,7 +276,6 @@ class FileProvider {
   }
 
   resolveDirectory (path, callback) {
-    if (!path) return callback(null, { [this.type]: {} })
     path = this.removePrefix(path)
     if (path.indexOf('/') !== 0) path = '/' + path
 

libs/remix-ui/plugin-manager/src/lib/components/LocalPluginForm.tsx

@@ -5,7 +5,7 @@ import { Toaster } from '@remix-ui/toaster'
 import { IframePlugin, WebsocketPlugin } from '@remixproject/engine-web'
 
 import { localPluginReducerActionType, localPluginToastReducer } from '../reducers/pluginManagerReducer'
-import { FormStateProps, PluginManagerComponent } from '../../types'
+import { canActivate, FormStateProps, PluginManagerComponent } from '../../types'
 
 interface LocalPluginFormProps {
   closeModal: () => void

package.json

@@ -136,13 +136,13 @@
     "@ethereumjs/common": "^2.4.0",
     "@ethereumjs/tx": "^3.3.0",
     "@ethereumjs/vm": "^5.5.0",
-    "@remixproject/engine": "^0.3.17",
+    "@remixproject/engine": "^0.3.20-alpha.2",
-    "@remixproject/engine-web": "^0.3.17",
+    "@remixproject/engine-web": "^0.3.20-alpha.2",
-    "@remixproject/plugin": "^0.3.17",
+    "@remixproject/plugin": "^0.3.20-alpha.2",
-    "@remixproject/plugin-api": "^0.3.17",
+    "@remixproject/plugin-api": "^0.3.20-alpha.2",
-    "@remixproject/plugin-utils": "^0.3.17",
+    "@remixproject/plugin-utils": "^0.3.20-alpha.2",
-    "@remixproject/plugin-webview": "^0.3.17",
+    "@remixproject/plugin-webview": "^0.3.20-alpha.2",
-    "@remixproject/plugin-ws": "^0.3.17",
+    "@remixproject/plugin-ws": "^0.3.20-alpha.2",
     "ansi-gray": "^0.1.1",
     "async": "^2.6.2",
     "axios": ">=0.21.1",