Fetch and compile before debugging

package.json

@@ -55,13 +55,13 @@
     "npm-merge-driver": "^2.3.5",
     "npm-run-all": "^4.0.2",
     "onchange": "^3.2.1",
-    "remix-analyzer": "0.5.0-beta.2",
-    "remix-debug": "0.3.28",
-    "remix-lib": "0.4.24",
-    "remix-simulator": "0.1.9-beta.2",
-    "remix-solidity": "0.3.27",
+    "remix-analyzer": "0.5.2",
+    "remix-debug": "0.4.4",
+    "remix-lib": "0.4.29",
+    "remix-simulator": "0.1.9-beta.5",
+    "remix-solidity": "0.3.30",
     "remix-tabs": "1.0.48",
-    "remix-tests": "0.1.30",
+    "remix-tests": "0.1.33",
     "remixd": "0.1.8-alpha.10",
     "request": "^2.83.0",
     "rimraf": "^2.6.1",

src/app.js

@@ -54,6 +54,7 @@ import { HiddenPanel } from './app/components/hidden-panel'
 import { VerticalIcons } from './app/components/vertical-icons'
 import { LandingPage } from './app/ui/landing-page/landing-page'
 import { MainPanel } from './app/components/main-panel'
+import FetchAndCompile from './app/compiler/compiler-sourceVerifier-fetchAndCompile'
 
 import migrateFileSystem from './migrateFileSystem'
 
@@ -267,6 +268,8 @@ Please make a backup of your contracts and start using http://remix.ethereum.org
   const compilersArtefacts = new CompilersArtefacts() // store all the compilation results (key represent a compiler name)
   registry.put({api: compilersArtefacts, name: 'compilersartefacts'})
 
+  // service which fetch contract artifacts from sourve-verify, put artifacts in remix and compile it
+  const fetchAndCompile = new FetchAndCompile()
   // ----------------- network service (resolve network id / name) -----
   const networkModule = new NetworkModule(blockchain)
   // ----------------- represent the current selected web3 provider ----
@@ -305,7 +308,8 @@ Please make a backup of your contracts and start using http://remix.ethereum.org
     offsetToLineColumnConverter,
     contextualListener,
     terminal,
-    web3Provider
+    web3Provider,
+    fetchAndCompile
   ])
 
   // LAYOUT & SYSTEM VIEWS
@@ -391,7 +395,7 @@ Please make a backup of your contracts and start using http://remix.ethereum.org
 
   await appManager.activatePlugin(['contentImport', 'theme', 'editor', 'fileManager', 'compilerMetadata', 'compilerArtefacts', 'network', 'web3Provider', 'offsetToLineColumnConverter'])
   await appManager.activatePlugin(['mainPanel', 'menuicons'])
-  await appManager.activatePlugin(['home', 'sidePanel', 'hiddenPanel', 'pluginManager', 'fileExplorers', 'settings', 'contextualListener', 'scriptRunner', 'terminal'])
+  await appManager.activatePlugin(['home', 'sidePanel', 'hiddenPanel', 'pluginManager', 'fileExplorers', 'settings', 'contextualListener', 'scriptRunner', 'terminal', 'fetchAndCompile'])
 
   // Set workspace after initial activation
   if (Array.isArray(workspace)) await appManager.activatePlugin(workspace)

src/app/compiler/compiler-artefacts.js

@@ -33,4 +33,16 @@ module.exports = class CompilerArtefacts extends Plugin {
       this.compilersArtefacts['__last'] = new CompilerAbstract(languageVersion, data, source)
     })
   }
+
+  addResolvedContract (address, compilerData) {
+    this.compilersArtefacts[address] = compilerData
+  }
+
+  isResolved (address) {
+    return this.compilersArtefacts[address] !== undefined
+  }
+
+  get (key) {
+    return this.compilersArtefacts[key]
+  }
 }

src/app/compiler/compiler-helpers.js

@@ -0,0 +1,20 @@
+'use strict'
+import { canUseWorker } from './compiler-utils'
+import { Compiler } from 'remix-solidity'
+import CompilerAbstract from './compiler-abstract'
+
+export const compile = async (compilationTargets, settings) => {
+  return await (() => {
+    return new Promise((resolve, reject) => {
+      const compiler = new Compiler(() => {})
+      compiler.set('evmVersion', settings.evmVersion)
+      compiler.set('optimize', settings.optimize)
+      compiler.loadVersion(canUseWorker(settings.version), settings.compilerUrl)
+      compiler.event.register('compilationFinished', (success, compilationData, source) => {
+        if (!success) return reject(compilationData)
+        resolve(new CompilerAbstract(settings.version, compilationData, source))
+      })
+      compiler.event.register('compilerLoaded', _ => compiler.compile(compilationTargets, ''))
+    })
+  })()
+}

src/app/compiler/compiler-sourceVerifier-fetchAndCompile.js

@@ -0,0 +1,125 @@
+const ethutil = require('ethereumjs-util')
+import * as packageJson from '../../../package.json'
+import { Plugin } from '@remixproject/engine'
+import { urlFromVersion } from './compiler-utils'
+import { compile } from './compiler-helpers'
+import globalRegistry from '../../global/registry'
+
+import remixLib from 'remix-lib'
+
+const profile = {
+  name: 'fetchAndCompile',
+  methods: ['resolve'],
+  version: packageJson.version
+}
+
+export default class FetchAndCompile extends Plugin {
+
+  constructor () {
+    super(profile)
+    this.unresolvedAddresses = []
+    this.sourceVerifierNetWork = ['Main', 'Rinkeby', 'Ropsten', 'Goerli']
+  }
+
+  /**
+   * Fetch compiliation metadata from source-Verify from a given @arg contractAddress - https://github.com/ethereum/source-verify
+   * Put the artifacts in the file explorer
+   * Compile the code using Solidity compiler
+   * Returns compilation data
+   *
+   * @param {string} contractAddress - Address of the contrac to resolve
+   * @param {string} compilersartefacts - Object containing a mapping of compilation results (byContractAddress and __last)
+   * @return {CompilerAbstract} - compilation data targeting the given @arg contractAddress
+   */
+  async resolve (contractAddress, targetPath, web3) {
+    contractAddress = ethutil.toChecksumAddress(contractAddress)
+    const compilersartefacts = globalRegistry.get('compilersartefacts').api
+
+    const localCompilation = () => compilersartefacts.get('__last') ? compilersartefacts.get('__last') : null
+
+    const resolved = compilersartefacts.get(contractAddress)
+    if (resolved) return resolved
+    if (this.unresolvedAddresses.includes(contractAddress)) return localCompilation()
+
+    // sometimes when doing an internal call, the only available artifact is the Solidity interface.
+    // resolving addresses of internal call would allow to step over the source code, even if the declaration was made using an Interface.
+
+    let network
+    try {
+      network = await this.call('network', 'detectNetwork')
+    } catch (e) {
+      return localCompilation()
+    }
+    if (!network) return localCompilation()
+    if (!this.sourceVerifierNetWork.includes(network.name)) return localCompilation()
+
+    // check if the contract if part of the local compilation result
+    const codeAtAddress = await web3.eth.getCode(contractAddress)
+    const compilation = localCompilation()
+    if (compilation) {
+      let found = false
+      compilation.visitContracts((contract) => {
+        found = remixLib.util.compareByteCode('0x' + contract.object.evm.deployedBytecode.object, codeAtAddress)
+        return found
+      })
+      if (found) {
+        compilersartefacts.addResolvedContract(contractAddress, compilation)
+        setTimeout(_ => this.emit('usingLocalCompilation', contractAddress), 0)
+        return compilation
+      }
+    }
+
+    let name = network.name.toLowerCase()
+    name === 'main' ? 'mainnet' : name // source-verifier api expect "mainnet" and not "main"
+    let data
+    try {
+      data = await this.call('source-verification', 'fetch', contractAddress, name.toLowerCase())
+    } catch (e) {
+      setTimeout(_ => this.emit('sourceVerificationNotAvailable'), 0)
+      this.unresolvedAddresses.push(contractAddress)
+      return localCompilation()
+    }
+    if (!data || !data.metadata) {
+      setTimeout(_ => this.emit('notFound', contractAddress), 0)
+      this.unresolvedAddresses.push(contractAddress)
+      return localCompilation()
+    }
+
+    // set the solidity contract code using metadata
+    await this.call('fileManager', 'setFile', `${targetPath}/${name}/${contractAddress}/metadata.json`, JSON.stringify(data.metadata, null, '\t'))
+    let compilationTargets = {}
+    for (let file in data.metadata.sources) {
+      const urls = data.metadata.sources[file].urls
+      for (let url of urls) {
+        if (url.includes('ipfs')) {
+          let stdUrl = `ipfs://${url.split('/')[2]}`
+          const source = await this.call('contentImport', 'resolve', stdUrl)
+          file = file.replace('browser/', '') // should be fixed in the remix IDE end.
+          const path = `${targetPath}/${name}/${contractAddress}/${file}`
+          await this.call('fileManager', 'setFile', path, source.content)
+          compilationTargets[path] = { content: source.content }
+          break
+        }
+      }
+    }
+
+    // compile
+    const settings = {
+      version: data.metadata.compiler.version,
+      languageName: data.metadata.language,
+      evmVersion: data.metadata.settings.evmVersion,
+      optimize: data.metadata.settings.optimizer.enabled,
+      compilerUrl: urlFromVersion(data.metadata.compiler.version)
+    }
+    try {
+      setTimeout(_ => this.emit('compiling', settings), 0)
+      const compData = await compile(compilationTargets, settings)
+      compilersartefacts.addResolvedContract(contractAddress, compData)
+      return compData
+    } catch (e) {
+      this.unresolvedAddresses.push(contractAddress)
+      setTimeout(_ => this.emit('compilationFailed'), 0)
+      return localCompilation()
+    }
+  }
+}

src/app/compiler/compiler-utils.js

@@ -1,6 +1,11 @@
 const semver = require('semver')
 /* global Worker */
 
+export const baseUrl = 'https://solc-bin.ethereum.org/bin'
+
+export function urlFromVersion (version) {
+  return `${baseUrl}/soljson-v${version}.js`
+}
 /**
  * Checks if the worker can be used to load a compiler.
  * checks a compiler whitelist, browser support and OS.

src/app/tabs/debugger-tab.js

@@ -1,8 +1,8 @@
-var yo = require('yo-yo')
-var css = require('./styles/debugger-tab-styles')
-
-var DebuggerUI = require('./debugger/debuggerUI')
-
+const yo = require('yo-yo')
+const remixLib = require('remix-lib')
+const css = require('./styles/debugger-tab-styles')
+import toaster from '../ui/tooltip'
+const DebuggerUI = require('./debugger/debuggerUI')
 import { ViewPlugin } from '@remixproject/engine'
 import * as packageJson from '../../../package.json'
 
@@ -34,7 +34,38 @@ class DebuggerTab extends ViewPlugin {
       <div class="${css.debuggerTabView}" id="debugView">
         <div id="debugger" class="${css.debugger}"></div>
       </div>`
-    this.debuggerUI = new DebuggerUI(this.el.querySelector('#debugger'), this.blockchain)
+
+    this.on('fetchAndCompile', 'compiling', (settings) => {
+      toaster(yo`<div><b>Recompiling and debugging with params</b><pre class="text-left">${JSON.stringify(settings, null, '\t')}</pre></div>`)
+    })
+
+    this.on('fetchAndCompile', 'compilationFailed', (data) => {
+      toaster(yo`<div><b>Compilation failed...</b> continuing <i>without</i> source code debugging.</div>`)
+    })
+
+    this.on('fetchAndCompile', 'notFound', (contractAddress) => {
+      toaster(yo`<div><b>Contract ${contractAddress} not found in source code repository</b> continuing <i>without</i> source code debugging.</div>`)
+    })
+
+    this.on('fetchAndCompile', 'usingLocalCompilation', (contractAddress) => {
+      toaster(yo`<div><b>Using compilation result from Solidity module</b></div>`)
+    })
+
+    this.on('fetchAndCompile', 'sourceVerificationNotAvailable', () => {
+      toaster(yo`<div><b>Source verification plugin not activated or not available.</b> continuing <i>without</i> source code debugging.</div>`)
+    })
+
+    this.debuggerUI = new DebuggerUI(
+      this.el.querySelector('#debugger'),
+      this.blockchain,
+      (address, receipt) => {
+        const target = (address && remixLib.helpers.trace.isContractCreation(address)) ? receipt.contractAddress : address
+        return this.call('fetchAndCompile', 'resolve', target || receipt.contractAddress || receipt.to, '.debug', this.blockchain.web3())
+      })
+
+    this.call('manager', 'activatePlugin', 'source-verification')
+    // this.call('manager', 'activatePlugin', 'udapp')
+
     return this.el
   }
 

src/app/tabs/debugger/debuggerUI.js

@@ -30,9 +30,10 @@ var css = csjs`
 
 class DebuggerUI {
 
-  constructor (container, blockchain) {
+  constructor (container, blockchain, fetchContractAndCompile) {
     this.registry = globalRegistry
     this.blockchain = blockchain
+    this.fetchContractAndCompile = fetchContractAndCompile
     this.event = new EventManager()
 
     this.isActive = false
@@ -43,6 +44,7 @@ class DebuggerUI {
     this.stepManager = null
 
     this.statusMessage = ''
+    this.currentReceipt
 
     this.view
 
@@ -77,8 +79,14 @@ class DebuggerUI {
       self.isActive = isActive
     })
 
-    this.debugger.event.register('newSourceLocation', function (lineColumnPos, rawLocation) {
-      self.sourceHighlighter.currentSourceLocation(lineColumnPos, rawLocation)
+    this.debugger.event.register('newSourceLocation', async function (lineColumnPos, rawLocation) {
+      const contracts = await self.fetchContractAndCompile(
+        self.currentReceipt.contractAddress || self.currentReceipt.to,
+        self.currentReceipt)
+      if (contracts) {
+        const path = contracts.getSourceName(rawLocation.file)
+        if (path) self.sourceHighlighter.currentSourceLocationFromfileName(lineColumnPos, path)
+      }
     })
 
     this.debugger.event.register('debuggerUnloaded', self.unLoad.bind(this))
@@ -122,15 +130,19 @@ class DebuggerUI {
   async startDebugging (blockNumber, txNumber, tx) {
     if (this.debugger) this.unLoad()
 
-    let compilers = this.registry.get('compilersartefacts').api
-    let lastCompilationResult
-    if (compilers['__last']) lastCompilationResult = compilers['__last']
-
     let web3 = await this.getDebugWeb3()
+    this.currentReceipt = await web3.eth.getTransactionReceipt(txNumber)
     this.debugger = new Debugger({
       web3,
       offsetToLineColumnConverter: this.registry.get('offsettolinecolumnconverter').api,
-      compiler: { lastCompilationResult }
+      compilationResult: async (address) => {
+        try {
+          return await this.fetchContractAndCompile(address, this.currentReceipt)
+        } catch (e) {
+          console.error(e)
+        }
+        return null
+      }
     })
 
     this.listenToEvents()
@@ -147,16 +159,20 @@ class DebuggerUI {
 
   getTrace (hash) {
     return new Promise(async (resolve, reject) => {
-      const compilers = this.registry.get('compilersartefacts').api
-      let lastCompilationResult
-      if (compilers['__last']) lastCompilationResult = compilers['__last']
-
       const web3 = await this.getDebugWeb3()
 
+      this.currentReceipt = await web3.eth.getTransactionReceipt(hash)
       const debug = new Debugger({
         web3,
         offsetToLineColumnConverter: this.registry.get('offsettolinecolumnconverter').api,
-        compiler: { lastCompilationResult }
+        compilationResult: async (address) => {
+          try {
+            return await this.fetchContractAndCompile(address, this.currentReceipt)
+          } catch (e) {
+            console.error(e)
+          }
+          return null
+        }
       })
       debug.debugger.traceManager.traceRetriever.getTrace(hash, (error, trace) => {
         if (error) return reject(error)

src/app/tabs/debugger/debuggerUI/TxBrowser.js

@@ -93,6 +93,7 @@ TxBrowser.prototype.render = function () {
       type='text'
       oninput=${this.txInputChanged.bind(this)}
       placeholder=${'Transaction hash, should start with 0x'}
+      data-id="debuggerTransactionInput"
     />
   `
   let txButton = yo`
@@ -101,6 +102,7 @@ TxBrowser.prototype.render = function () {
       id='load'
       title='${this.state.debugging ? 'Stop' : 'Start'} debugging'
       onclick=${function () { self.submit() }}
+      data-id="debuggerTransactionStartButton"
     >
       ${this.state.debugging ? 'Stop' : 'Start'} debugging
     </button>