support submodules

apps/remix-ide/src/app/files/dgitProvider.js

@@ -21,7 +21,7 @@ const profile = {
   description: 'Decentralized git provider',
   icon: 'assets/img/fileManager.webp',
   version: '0.0.1',
-  methods: ['init', 'localStorageUsed', 'addremote', 'delremote', 'remotes', 'fetch', 'clone', 'export', 'import', 'status', 'log', 'commit', 'add', 'remove', 'rm', 'lsfiles', 'readblob', 'resolveref', 'branches', 'branch', 'checkout', 'currentbranch', 'push', 'pin', 'pull', 'pinList', 'unPin', 'setIpfsConfig', 'zip', 'setItem', 'getItem'],
+  methods: ['init', 'localStorageUsed', 'addremote', 'delremote', 'remotes', 'fetch', 'clone', 'export', 'import', 'status', 'log', 'commit', 'add', 'remove', 'rm', 'lsfiles', 'readblob', 'resolveref', 'branches', 'branch', 'checkout', 'currentbranch', 'push', 'pin', 'pull', 'pinList', 'unPin', 'setIpfsConfig', 'zip', 'setItem', 'getItem', 'updateSubmodules'],
   kind: 'file-system'
 }
 class DGitProvider extends Plugin {
@@ -48,13 +48,13 @@ class DGitProvider extends Plugin {
     this.ipfsSources = [this.remixIPFS, this.globalIPFSConfig, this.ipfsconfig]
   }
 
-  async getGitConfig () {
+  async getGitConfig (dir = '') {
     const workspace = await this.call('filePanel', 'getCurrentWorkspace')
 
     if (!workspace) return
     return {
       fs: window.remixFileSystemCallback,
-      dir: addSlash(workspace.absolutePath)
+      dir: addSlash(path.join(workspace.absolutePath, dir || '')),
     }
   }
 
@@ -105,15 +105,37 @@ class DGitProvider extends Plugin {
   }
 
   async checkout (cmd, refresh = true) {
+    const gitmodules = await this.parseGitmodules() || []
     await git.checkout({
       ...await this.getGitConfig(),
       ...cmd
     })
-    if (refresh) {
+    const newgitmodules = await this.parseGitmodules() || []
+    // find the difference between the two gitmodule versions
+    const toRemove = gitmodules.filter((module) => {
+      return !newgitmodules.find((newmodule) => {
+        return newmodule.name === module.name
+      })
+    })
+
+    for (const module of toRemove) {
+      const path = (await this.getGitConfig(module.path)).dir
+      if (await window.remixFileSystem.exists(path)) {
+        const stat = await window.remixFileSystem.stat(path)
+        try {
+          if (stat.isDirectory()) {
+            await window.remixFileSystem.unlink((await this.getGitConfig(module.path)).dir)
+          }
+        } catch (e) {
+        }
+      }
+    }
+
+    if (refresh)
       setTimeout(async () => {
         await this.call('fileManager', 'refresh')
       }, 1000)
-    }
+
     this.emit('checkout')
   }
 
@@ -150,19 +172,19 @@ class DGitProvider extends Plugin {
   }
 
   async currentbranch (config) {
-    try{
+    try {
       const defaultConfig = await this.getGitConfig()
       const cmd = config ? defaultConfig ? { ...defaultConfig, ...config } : config : defaultConfig
       const name = await git.currentBranch(cmd)
 
       return name
-    }catch(e){
+    } catch (e) {
       return ''
     }
   }
 
   async branches (config) {
-    try{
+    try {
       const defaultConfig = await this.getGitConfig()
       const cmd = config ? defaultConfig ? { ...defaultConfig, ...config } : config : defaultConfig
       const remotes = await this.remotes(config)
@@ -174,7 +196,7 @@ class DGitProvider extends Plugin {
         branches = [...branches, ...remotebranches]
       }
       return branches
-    }catch(e){
+    } catch (e) {
       return []
     }
   }
@@ -261,6 +283,7 @@ class DGitProvider extends Plugin {
     }
 
     const result = await git.clone(cmd)
+    await this.updateSubmodules(input)
     if (!workspaceExists) {
       setTimeout(async () => {
         await this.call('fileManager', 'refresh')
@@ -270,7 +293,80 @@ class DGitProvider extends Plugin {
     return result
   }
 
-  async push (input) {
+  async parseGitmodules () {
+    try {
+      const gitmodules = await this.call('fileManager', 'readFile', '.gitmodules')
+      if (gitmodules) {
+        const lines = gitmodules.split('\n')
+        let currentModule = {}
+        let modules = []
+        for (let line of lines) {
+          line = line.trim()
+          if (line.startsWith('[')) {
+            if (currentModule.path) {
+              modules.push(currentModule)
+            }
+            currentModule = {}
+            currentModule.name = line.replace('[submodule "', '').replace('"]', '')
+          } else if (line.startsWith('url')) {
+            currentModule.url = line.replace('url = ', '')
+          } else if (line.startsWith('path')) {
+            currentModule.path = line.replace('path = ', '')
+          }
+        }
+        if (currentModule.path) {
+          modules.push(currentModule)
+        }
+        return modules
+      }
+    } catch (e) {
+      // do nothing
+    }
+  }
+
+  async updateSubmodules(input) {
+    try {
+      const gitmodules = await this.parseGitmodules()
+      //parse gitmodules
+      if (gitmodules) {
+        for (let module of gitmodules) {
+          const path = (await this.getGitConfig(module.path)).dir
+          if (await window.remixFileSystem.exists(path)) {
+            const stat = await window.remixFileSystem.stat(path)
+            try {
+              if (stat.isDirectory()) {
+                await window.remixFileSystem.unlink((await this.getGitConfig(module.path)).dir)
+              }
+            } catch (e) {
+            }
+          }
+        }
+        for (let module of gitmodules) {
+          try {
+            const cmd = {
+              url: module.url,
+              singleBranch: true,
+              depth: 1,
+              ...await this.parseInput(input),
+              ...await this.getGitConfig(module.path)
+            }
+            await git.clone(cmd)
+          } catch (e) {
+            console.log(e)
+
+          }
+        }
+        setTimeout(async () => {
+          await this.call('fileManager', 'refresh')
+        }, 1000)
+      }
+    } catch (e) {
+      console.log(e)
+      // do nothing
+    }
+  }
+
+  async push(input) {
     const cmd = {
       force: input.force,
       ref: input.ref,

apps/remix-ide/src/app/files/fileManager.ts

@@ -25,7 +25,7 @@ const profile = {
   methods: ['closeAllFiles', 'closeFile', 'file', 'exists', 'open', 'writeFile', 'writeMultipleFiles', 'writeFileNoRewrite',
     'readFile', 'copyFile', 'copyDir', 'rename', 'mkdir', 'readdir', 'dirList', 'fileList', 'remove', 'getCurrentFile', 'getFile',
     'getFolder', 'setFile', 'switchFile', 'refresh', 'getProviderOf', 'getProviderByName', 'getPathFromUrl', 'getUrlFromPath',
-    'saveCurrentFile', 'setBatchFiles', 'isGitRepo', 'isFile', 'isDirectory'
+    'saveCurrentFile', 'setBatchFiles', 'isGitRepo', 'isFile', 'isDirectory', 'hasGitSubmodule'
   ],
   kind: 'file-system'
 }
@@ -919,6 +919,13 @@ class FileManager extends Plugin {
     return exists
   }
 
+  async hasGitSubmodules(): Promise<boolean> {
+    const path = '.gitmodules'
+    const exists = await this.exists(path)
+
+    return exists
+  }
+
 
   async moveFileIsAllowed (src: string, dest: string) {
     try {

libs/remix-ui/workspace/src/lib/actions/payload.ts

@@ -294,6 +294,13 @@ export const setCurrentWorkspaceIsGitRepo = (isRepo: boolean): Action<'SET_CURRE
   }
 }
 
+export const setCurrentWorkspaceHasGitSubmodules = (hasGitSubmodules: boolean): Action<'SET_CURRENT_WORKSPACE_HAS_GIT_SUBMODULES'> => {
+  return {
+    type: 'SET_CURRENT_WORKSPACE_HAS_GIT_SUBMODULES',
+    payload: hasGitSubmodules
+  }
+}
+
 export const setGitConfig = (config: {username: string, token: string, email: string}): Action<'SET_GIT_CONFIG'> => {
   return {
     type: 'SET_GIT_CONFIG',

libs/remix-ui/workspace/src/lib/actions/workspace.ts

@@ -25,6 +25,7 @@ import {
   setRenameWorkspace,
   setCurrentWorkspaceIsGitRepo,
   setGitConfig,
+  setCurrentWorkspaceHasGitSubmodules,
 } from './payload'
 import { addSlash, checkSlash, checkSpecialChars } from '@remix-ui/helper'
 
@@ -423,7 +424,7 @@ export const switchToWorkspace = async (name: string) => {
     await plugin.fileProviders.workspace.setWorkspace(name)
     await plugin.setWorkspace({ name, isLocalhost: false })
     const isGitRepo = await plugin.fileManager.isGitRepo()
-
+    const hasGitSubmodule = await plugin.fileManager.hasGitSubmodules()
     if (isGitRepo) {
       const isActive = await plugin.call('manager', 'isActive', 'dgit')
       if (!isActive) await plugin.call('manager', 'activatePlugin', 'dgit')
@@ -513,9 +514,9 @@ export const uploadFolder = async (target, targetFolder: string, cb?: (err: Erro
   }
 }
 
-export const getWorkspaces = async (): Promise<{ name: string; isGitRepo: boolean; branches?: { remote: any; name: string }[]; currentBranch?: string }[]> | undefined => {
+export const getWorkspaces = async (): Promise<{ name: string; isGitRepo: boolean; hasGitSubmodules: boolean; branches?: { remote: any; name: string }[]; currentBranch?: string }[]> | undefined => {
   try {
-    const workspaces: { name: string; isGitRepo: boolean; branches?: { remote: any; name: string }[]; currentBranch?: string }[] = await new Promise((resolve, reject) => {
+    const workspaces: { name: string; isGitRepo: boolean; hasGitSubmodules: boolean; branches?: { remote: any; name: string }[]; currentBranch?: string }[] = await new Promise((resolve, reject) => {
       const workspacesPath = plugin.fileProviders.workspace.workspacesPath
 
       plugin.fileProviders.browser.resolveDirectory('/' + workspacesPath, (error, items) => {
@@ -527,7 +528,7 @@ export const getWorkspaces = async (): Promise<{ name: string; isGitRepo: boolea
             .filter((item) => items[item].isDirectory)
             .map(async (folder) => {
               const isGitRepo: boolean = await plugin.fileProviders.browser.exists('/' + folder + '/.git')
-
+              const hasGitSubmodules: boolean = await plugin.fileProviders.browser.exists('/' + folder + '/.gitmodules')
               if (isGitRepo) {
                 let branches = []
                 let currentBranch = null
@@ -539,11 +540,13 @@ export const getWorkspaces = async (): Promise<{ name: string; isGitRepo: boolea
                   isGitRepo,
                   branches,
                   currentBranch,
+                  hasGitSubmodules
                 }
               } else {
                 return {
                   name: folder.replace(workspacesPath + '/', ''),
                   isGitRepo,
+                  hasGitSubmodules
                 }
               }
             })
@@ -608,7 +611,9 @@ export const cloneRepository = async (url: string) => {
 
 export const checkGit = async () => {
   const isGitRepo = await plugin.fileManager.isGitRepo()
+  const hasGitSubmodule = await plugin.fileManager.hasGitSubmodules()
   dispatch(setCurrentWorkspaceIsGitRepo(isGitRepo))
+  dispatch(setCurrentWorkspaceHasGitSubmodules(hasGitSubmodule))
   await refreshBranches()
   const currentBranch = await plugin.call('dGitProvider', 'currentbranch')
   dispatch(setCurrentWorkspaceCurrentBranch(currentBranch))
@@ -773,6 +778,12 @@ export const createHelperScripts = async (script: string) => {
   plugin.call('notification', 'toast', 'scripts added in the "scripts" folder')
 }
 
+export const updateGitSubmodules = async () => {
+  dispatch(cloneRepositoryRequest())
+  await plugin.call('dGitProvider', 'updateSubmodules')
+  dispatch(cloneRepositorySuccess())
+}
+
 export const checkoutRemoteBranch = async (branch: string, remote: string) => {
   const localChanges = await hasLocalChanges()
 

libs/remix-ui/workspace/src/lib/contexts/index.ts

@@ -46,6 +46,7 @@ export const FileSystemContext = createContext<{
   dispatchCreateTsSolGithubAction: () => Promise<void>,
   dispatchCreateSlitherGithubAction: () => Promise<void>
   dispatchCreateHelperScripts: (script: string) => Promise<void>
+  dispatchUpdateGitSubmodules: () => Promise<void>
     }>(null)
   
     

libs/remix-ui/workspace/src/lib/providers/FileSystemProvider.tsx

@@ -45,7 +45,8 @@ import {
   createSolidityGithubAction,
   createTsSolGithubAction,
   createSlitherGithubAction,
-  createHelperScripts
+  createHelperScripts,
+  updateGitSubmodules
 } from '../actions'
 import {Modal, WorkspaceProps, WorkspaceTemplate} from '../types'
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
@@ -224,6 +225,10 @@ export const FileSystemProvider = (props: WorkspaceProps) => {
     await createHelperScripts(script)
   }
 
+  const dispatchUpdateGitSubmodules = async () => {
+    await updateGitSubmodules()
+  }
+
   useEffect(() => {
     dispatchInitWorkspace()
   }, [])
@@ -341,7 +346,8 @@ export const FileSystemProvider = (props: WorkspaceProps) => {
     dispatchCreateSolidityGithubAction,
     dispatchCreateTsSolGithubAction,
     dispatchCreateSlitherGithubAction,
-    dispatchCreateHelperScripts
+    dispatchCreateHelperScripts,
+    dispatchUpdateGitSubmodules
   }
   return (
     <FileSystemContext.Provider value={value}>

libs/remix-ui/workspace/src/lib/reducers/workspace.ts

@@ -9,6 +9,7 @@ export interface BrowserState {
     workspaces: {
       name: string
       isGitRepo: boolean
+      hasGitSubmodules?: boolean
       branches?: {
         remote: any
         name: string
@@ -807,6 +808,22 @@ export const browserReducer = (state = browserInitialState, action: Actions) =>
     }
   }
 
+  case 'SET_CURRENT_WORKSPACE_HAS_GIT_SUBMODULES': {
+    const payload = action.payload
+
+    return {
+      ...state,
+      browser: {
+        ...state.browser,
+        workspaces: state.browser.workspaces.map((workspace) => {
+          if (workspace.name === state.browser.currentWorkspace)
+            workspace.hasGitSubmodules = payload
+          return workspace
+        })
+      }
+    }
+  }
+
   case 'SET_GIT_CONFIG': {
     const payload =
         action.payload

libs/remix-ui/workspace/src/lib/remix-ui-workspace.tsx

@@ -24,6 +24,7 @@ export function Workspace() {
   const [selectedWorkspace, setSelectedWorkspace] = useState<{
     name: string
     isGitRepo: boolean
+    hasGitSubmodules?: boolean
     branches?: {remote: any; name: string}[]
     currentBranch?: string
   }>(null)
@@ -139,7 +140,6 @@ export function Workspace() {
 
   useEffect(() => {
     const workspace = global.fs.browser.workspaces.find((workspace) => workspace.name === currentWorkspace)
-
     setSelectedWorkspace(workspace)
   }, [currentWorkspace])
 
@@ -649,6 +649,14 @@ export function Workspace() {
     setShowBranches(isOpen)
   }
 
+  const updateSubModules = async () => {
+    try {
+      await global.dispatchUpdateGitSubmodules()
+    } catch (e) {
+      console.error(e)
+    }
+  }
+
   const handleBranchFilterChange = (e: ChangeEvent<HTMLInputElement>) => {
     const branchFilter = e.target.value
 
@@ -1122,6 +1130,12 @@ export function Workspace() {
         <div className={`bg-light border-top ${selectedWorkspace.isGitRepo && currentBranch ? 'd-block' : 'd-none'}`} data-id="workspaceGitPanel">
           <div className="d-flex justify-space-between p-1">
             <div className="mr-auto text-uppercase text-dark pt-2 pl-2">GIT</div>
+            {selectedWorkspace.hasGitSubmodules? 
+              <div className="pt-1 mr-1">
+                {global.fs.browser.isRequestingCloning ? <div style={{ height: 30 }} className='btn btn-sm border text-muted small'><i className="fad fa-spinner fa-spin"></i> updating submodules</div>  :
+                  <div style={{ height: 30 }} onClick={updateSubModules} className='btn btn-sm border text-muted small'>update submodules</div>}
+              </div>  
+              : null}
             <div className="pt-1 mr-1" data-id="workspaceGitBranchesDropdown">
               <Dropdown style={{height: 30, minWidth: 80}} onToggle={toggleBranches} show={showBranches} drop={'up'}>
                 <Dropdown.Toggle

libs/remix-ui/workspace/src/lib/types/index.ts

@@ -292,6 +292,7 @@ export interface ActionPayloadTypes {
   SET_CURRENT_WORKSPACE_BRANCHES: { remote: string | undefined; name: string }[],
   SET_CURRENT_WORKSPACE_CURRENT_BRANCH: string,
   SET_CURRENT_WORKSPACE_IS_GITREPO: boolean,
+  SET_CURRENT_WORKSPACE_HAS_GIT_SUBMODULES: boolean,
   SET_GIT_CONFIG: {
     username: string;
     token: string;

package.json

@@ -174,7 +174,7 @@
     "http-server": "^14.1.1",
     "intro.js": "^4.1.0",
     "isbinaryfile": "^3.0.2",
-    "isomorphic-git": "^1.8.2",
+    "isomorphic-git": "^1.25.0",
     "jquery": "^3.3.1",
     "js-yaml": "^4.1.0",
     "jspdf": "^2.5.1",

yarn.lock

@@ -11266,13 +11266,6 @@ decompress-response@^3.3.0:
   dependencies:
     mimic-response "^1.0.0"
 
-decompress-response@^4.2.0:
-  version "4.2.1"
-  resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-4.2.1.tgz#414023cc7a302da25ce2ec82d0d5238ccafd8986"
-  integrity sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==
-  dependencies:
-    mimic-response "^2.0.0"
-
 decompress-response@^6.0.0:
   version "6.0.0"
   resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-6.0.0.tgz#ca387612ddb7e104bd16d85aab00d5ecf09c66fc"
@@ -16502,10 +16495,10 @@ isobject@^3.0.0, isobject@^3.0.1:
   resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df"
   integrity sha1-TkMekrEalzFjaqH5yNHMvP2reN8=
 
-isomorphic-git@^1.8.2:
-  version "1.10.1"
-  resolved "https://registry.yarnpkg.com/isomorphic-git/-/isomorphic-git-1.10.1.tgz#2f3a3d2d41baf6a88e046a21e0527bc13a21d663"
-  integrity sha512-abbPpKkykIVDJ92rtYoD4AOuT5/7PABHR2fDBrsm7H0r2ZT+MGpPL/FynrEJM6nTcFSieaIDxnHNGhfHO/v+bA==
+isomorphic-git@^1.25.0:
+  version "1.25.0"
+  resolved "https://registry.yarnpkg.com/isomorphic-git/-/isomorphic-git-1.25.0.tgz#3a04d7e70f75ebdbb991f9fa87cfec90e3742c9f"
+  integrity sha512-F8X7z74gL+jN4bd6qB6a3Z0QQzonWPkiQ3nK/oFWlrc2pIwVM9Uksl3YMFh99ltswsqoCoOthgasybX08/fiGg==
   dependencies:
     async-lock "^1.1.0"
     clean-git-ref "^2.0.1"
@@ -16517,7 +16510,7 @@ isomorphic-git@^1.8.2:
     pify "^4.0.1"
     readable-stream "^3.4.0"
     sha.js "^2.4.9"
-    simple-get "^3.0.2"
+    simple-get "^4.0.1"
 
 isomorphic-textencoder@1.0.1:
   version "1.0.1"
@@ -19202,11 +19195,6 @@ mimic-response@^1.0.0, mimic-response@^1.0.1:
   resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-1.0.1.tgz#4923538878eef42063cb8a3e3b0798781487ab1b"
   integrity sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==
 
-mimic-response@^2.0.0:
-  version "2.1.0"
-  resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-2.1.0.tgz#d13763d35f613d09ec37ebb30bac0469c0ee8f43"
-  integrity sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==
-
 mimic-response@^3.1.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-3.1.0.tgz#2d1d59af9c1b129815accc2c46a022a5ce1fa3c9"
@@ -23952,6 +23940,11 @@ rlp@^2.2.4:
   dependencies:
     bn.js "^5.2.0"
 
+rlp@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/rlp/-/rlp-3.0.0.tgz#5a60725ca4314a3a165feecca1836e4f2c1e2343"
+  integrity sha512-PD6U2PGk6Vq2spfgiWZdomLvRGDreBLxi5jv5M8EpRo3pU6VEm31KO+HFxE18Q3vgqfDrQ9pZA3FP95rkijNKw==
+
 rollup-plugin-copy@^3.4.0:
   version "3.4.0"
   resolved "https://registry.yarnpkg.com/rollup-plugin-copy/-/rollup-plugin-copy-3.4.0.tgz#f1228a3ffb66ffad8606e2f3fb7ff23141ed3286"
@@ -24589,12 +24582,12 @@ simple-get@^2.5.1:
     once "^1.3.1"
     simple-concat "^1.0.0"
 
-simple-get@^3.0.2:
-  version "3.1.1"
-  resolved "https://registry.yarnpkg.com/simple-get/-/simple-get-3.1.1.tgz#cc7ba77cfbe761036fbfce3d021af25fc5584d55"
-  integrity sha512-CQ5LTKGfCpvE1K0n2us+kuMPbk/q0EKl82s4aheV9oXjFEz6W/Y7oQFVJuU6QG77hRT4Ghb5RURteF5vnWjupA==
+simple-get@^4.0.1:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/simple-get/-/simple-get-4.0.1.tgz#4a39db549287c979d352112fa03fd99fd6bc3543"
+  integrity sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==
   dependencies:
-    decompress-response "^4.2.0"
+    decompress-response "^6.0.0"
     once "^1.3.1"
     simple-concat "^1.0.0"