Skip to content
This repository has been archived by the owner on Feb 12, 2024. It is now read-only.

feat: ipns over dht #1725

Merged
merged 9 commits into from
Dec 6, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -127,10 +127,10 @@
"joi": "^13.4.0",
"joi-browser": "^13.4.0",
"joi-multiaddr": "^3.0.0",
"libp2p": "~0.24.0",
"libp2p": "~0.24.1",
"libp2p-bootstrap": "~0.9.3",
"libp2p-crypto": "~0.14.1",
"libp2p-kad-dht": "~0.11.1",
"libp2p-kad-dht": "~0.12.1",
"libp2p-keychain": "~0.3.3",
"libp2p-mdns": "~0.12.0",
"libp2p-mplex": "~0.8.4",
Expand Down
2 changes: 1 addition & 1 deletion src/core/components/init.js
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ module.exports = function init (self) {
(_, cb) => {
const offlineDatastore = new OfflineDatastore(self._repo)

self._ipns = new IPNS(offlineDatastore, self._repo, self._peerInfo, self._keychain, self._options)
self._ipns = new IPNS(offlineDatastore, self._repo.datastore, self._peerInfo, self._keychain, self._options)
cb(null, true)
},
// add empty unixfs dir object (go-ipfs assumes this exists)
Expand Down
11 changes: 11 additions & 0 deletions src/core/components/libp2p.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
const promisify = require('promisify-es6')
const get = require('lodash/get')
const defaultsDeep = require('@nodeutils/defaults-deep')
const ipnsUtils = require('../ipns/routing/utils')

module.exports = function libp2p (self) {
return {
Expand All @@ -16,6 +17,7 @@ module.exports = function libp2p (self) {

const defaultBundle = (opts) => {
const libp2pDefaults = {
datastore: opts.datastore,
peerInfo: opts.peerInfo,
peerBook: opts.peerBook,
config: {
Expand Down Expand Up @@ -43,6 +45,14 @@ module.exports = function libp2p (self) {
get(opts.config, 'relay.hop.active', false))
}
},
dht: {
validators: {
ipns: ipnsUtils.validator
},
selectors: {
ipns: ipnsUtils.selector
}
},
EXPERIMENTAL: {
dht: get(opts.options, 'EXPERIMENTAL.dht', false),
pubsub: get(opts.options, 'EXPERIMENTAL.pubsub', false)
Expand Down Expand Up @@ -72,6 +82,7 @@ module.exports = function libp2p (self) {
self._libp2pNode = libp2pBundle({
options: self._options,
config: config,
datastore: self._repo.datastore,
peerInfo: self._peerInfo,
peerBook: self._peerInfoBook
})
Expand Down
14 changes: 9 additions & 5 deletions src/core/components/start.js
Original file line number Diff line number Diff line change
Expand Up @@ -54,14 +54,18 @@ module.exports = (self) => {
ipnsStores.push(pubsubDs)
}

// NOTE: IPNS routing is being replaced by the local repo datastore while the IPNS over DHT is not ready
// When DHT is added, if local option enabled, should receive offlineDatastore as well
const offlineDatastore = new OfflineDatastore(self._repo)
ipnsStores.push(offlineDatastore)
// DHT should be added as routing if we are not running with local flag
// TODO: Need to change this logic once DHT is enabled by default, for now fallback to Offline datastore
if (get(self._options, 'EXPERIMENTAL.dht', false) && !self._options.local) {
ipnsStores.push(self._libp2pNode.dht)
} else {
const offlineDatastore = new OfflineDatastore(self._repo)
ipnsStores.push(offlineDatastore)
}

// Create ipns routing with a set of datastores
const routing = new TieredDatastore(ipnsStores)
self._ipns = new IPNS(routing, self._repo, self._peerInfo, self._keychain, self._options)
self._ipns = new IPNS(routing, self._repo.datastore, self._peerInfo, self._keychain, self._options)

self._bitswap = new Bitswap(
self._libp2pNode,
Expand Down
6 changes: 3 additions & 3 deletions src/core/ipns/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@ const path = require('./path')
const defaultRecordTtl = 60 * 1000

class IPNS {
constructor (routing, repo, peerInfo, keychain, options) {
this.publisher = new IpnsPublisher(routing, repo)
this.republisher = new IpnsRepublisher(this.publisher, repo, peerInfo, keychain, options)
constructor (routing, datastore, peerInfo, keychain, options) {
this.publisher = new IpnsPublisher(routing, datastore)
this.republisher = new IpnsRepublisher(this.publisher, datastore, peerInfo, keychain, options)
this.resolver = new IpnsResolver(routing)
this.cache = new Receptacle({ max: 1000 }) // Create an LRU cache with max 1000 items
this.routing = routing
Expand Down
86 changes: 49 additions & 37 deletions src/core/ipns/publisher.js
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,9 @@ const defaultRecordTtl = 60 * 60 * 1000

// IpnsPublisher is capable of publishing and resolving names to the IPFS routing system.
class IpnsPublisher {
constructor (routing, repo) {
constructor (routing, datastore) {
this._routing = routing
this._repo = repo
this._datastore = datastore
}

// publish record with a eol
Expand Down Expand Up @@ -56,7 +56,6 @@ class IpnsPublisher {
log.error(errMsg)
return callback(errcode(new Error(errMsg), 'ERR_INVALID_PEER_ID'))
}

const publicKey = peerId._pubKey

ipns.embedPublicKey(publicKey, record, (err, embedPublicKeyRecord) => {
Expand All @@ -74,9 +73,10 @@ class IpnsPublisher {

series([
(cb) => this._publishEntry(keys.routingKey, embedPublicKeyRecord || record, peerId, cb),
// Publish the public key if a public key cannot be extracted from the ID
// We will be able to deprecate this part in the future, since the public keys will be only in the peerId
(cb) => embedPublicKeyRecord ? this._publishPublicKey(keys.routingPubKey, publicKey, peerId, cb) : cb()
// Publish the public key to support old go-ipfs nodes that are looking for it in the routing
// We will be able to deprecate this part in the future, since the public keys will be only
// in IPNS record and the peerId.
(cb) => this._publishPublicKey(keys.routingPubKey, publicKey, peerId, cb)
], (err) => {
if (err) {
log.error(err)
Expand Down Expand Up @@ -159,50 +159,57 @@ class IpnsPublisher {
}

options = options || {}
const checkRouting = !(options.checkRouting === false)

this._repo.datastore.get(ipns.getLocalKey(peerId.id), (err, dsVal) => {
let result
const checkRouting = options.checkRouting !== false

this._datastore.get(ipns.getLocalKey(peerId.id), (err, dsVal) => {
if (err) {
if (err.code !== 'ERR_NOT_FOUND') {
const errMsg = `unexpected error getting the ipns record ${peerId.id} from datastore`

log.error(errMsg)
return callback(errcode(new Error(errMsg), 'ERR_UNEXPECTED_DATASTORE_RESPONSE'))
} else {
if (!checkRouting) {
return callback(null, null)
} else {
// TODO ROUTING - get from DHT
return callback(new Error('not implemented yet'))
}
}
}

if (Buffer.isBuffer(dsVal)) {
result = dsVal
} else {
const errMsg = `found ipns record that we couldn't convert to a value`
if (!checkRouting) {
return callback((errcode(err)))
}

log.error(errMsg)
return callback(errcode(new Error(errMsg), 'ERR_INVALID_IPNS_RECORD'))
}
// Try to get from routing
let keys
try {
keys = ipns.getIdKeys(peerId.toBytes())
} catch (err) {
log.error(err)
return callback(err)
}

// unmarshal data
try {
result = ipns.unmarshal(dsVal)
} catch (err) {
const errMsg = `found ipns record that we couldn't convert to a value`
this._routing.get(keys.routingKey.toBuffer(), (err, res) => {
if (err) {
return callback(err)
}

log.error(errMsg)
return callback(null, null)
// unmarshal data
this._unmarshalData(res, callback)
})
} else {
// unmarshal data
this._unmarshalData(dsVal, callback)
}

callback(null, result)
})
}

_unmarshalData (data, callback) {
let result
try {
result = ipns.unmarshal(data)
} catch (err) {
log.error(err)
return callback(errcode(err, 'ERR_INVALID_RECORD_DATA'))
}

callback(null, result)
}

_updateOrCreateRecord (privKey, value, validity, peerId, callback) {
if (!(PeerId.isPeerId(peerId))) {
const errMsg = `peerId received is not valid`
Expand All @@ -212,12 +219,17 @@ class IpnsPublisher {
}

const getPublishedOptions = {
checkRouting: false // TODO ROUTING - change to true
checkRouting: true
}

this._getPublished(peerId, getPublishedOptions, (err, record) => {
if (err) {
return callback(err)
if (err.code !== 'ERR_NOT_FOUND') {
const errMsg = `unexpected error when determining the last published IPNS record for ${peerId.id}`

log.error(errMsg)
return callback(errcode(new Error(errMsg), 'ERR_DETERMINING_PUBLISHED_RECORD'))
}
}

// Determinate the record sequence number
Expand All @@ -241,7 +253,7 @@ class IpnsPublisher {
const data = ipns.marshal(entryData)

// Store the new record
this._repo.datastore.put(ipns.getLocalKey(peerId.id), data, (err, res) => {
this._datastore.put(ipns.getLocalKey(peerId.id), data, (err, res) => {
if (err) {
const errMsg = `ipns record for ${value} could not be stored in the datastore`

Expand Down
6 changes: 3 additions & 3 deletions src/core/ipns/republisher.js
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@ const defaultBroadcastInterval = 4 * hour
const defaultRecordLifetime = 24 * hour

class IpnsRepublisher {
constructor (publisher, repo, peerInfo, keychain, options) {
constructor (publisher, datastore, peerInfo, keychain, options) {
this._publisher = publisher
this._repo = repo
this._datastore = datastore
this._peerInfo = peerInfo
this._keychain = keychain
this._options = options
Expand Down Expand Up @@ -160,7 +160,7 @@ class IpnsRepublisher {
return callback(errcode(new Error(errMsg), 'ERR_INVALID_PEER_ID'))
}

this._repo.datastore.get(ipns.getLocalKey(peerId.id), (err, dsVal) => {
this._datastore.get(ipns.getLocalKey(peerId.id), (err, dsVal) => {
// error handling
// no need to republish
if (err && err.notFound) {
Expand Down
66 changes: 50 additions & 16 deletions src/core/ipns/resolver.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
'use strict'

const ipns = require('ipns')
const crypto = require('libp2p-crypto')
const PeerId = require('peer-id')
const errcode = require('err-code')

Expand Down Expand Up @@ -96,13 +97,9 @@ class IpnsResolver {
return callback(err)
}

const { routingKey } = ipns.getIdKeys(peerId.toBytes())
const { routingKey, routingPubKey } = ipns.getIdKeys(peerId.toBytes())

// TODO DHT - get public key from routing?
// https://github.com/ipfs/go-ipfs/blob/master/namesys/routing.go#L70
// https://github.com/libp2p/go-libp2p-routing/blob/master/routing.go#L99

this._routing.get(routingKey.toBuffer(), (err, res) => {
this._routing.get(routingKey.toBuffer(), (err, record) => {
if (err) {
if (err.code !== 'ERR_NOT_FOUND') {
const errMsg = `unexpected error getting the ipns record ${peerId.id}`
Expand All @@ -116,29 +113,66 @@ class IpnsResolver {
return callback(errcode(new Error(errMsg), 'ERR_NO_RECORD_FOUND'))
}

// IPNS entry
let ipnsEntry
try {
ipnsEntry = ipns.unmarshal(res)
ipnsEntry = ipns.unmarshal(record)
} catch (err) {
const errMsg = `found ipns record that we couldn't convert to a value`

log.error(errMsg)
return callback(errcode(new Error(errMsg), 'ERR_INVALID_RECORD_RECEIVED'))
}

ipns.extractPublicKey(peerId, ipnsEntry, (err, pubKey) => {
// if the record has a public key validate it
if (ipnsEntry.pubKey) {
return this._validateRecord(peerId, ipnsEntry, callback)
}

// Otherwise, try to get the public key from routing
this._routing.get(routingKey.toBuffer(), (err, pubKey) => {
if (err) {
return callback(err)
}
if (err.code !== 'ERR_NOT_FOUND') {
const errMsg = `unexpected error getting the public key for the ipns record ${peerId.id}`

// IPNS entry validation
ipns.validate(pubKey, ipnsEntry, (err) => {
if (err) {
return callback(err)
log.error(errMsg)
return callback(errcode(new Error(errMsg), 'ERR_UNEXPECTED_ERROR_GETTING_PUB_KEY'))
}
const errMsg = `public key requested was not found for ${name} (${routingPubKey}) in the network`

log.error(errMsg)
return callback(errcode(new Error(errMsg), 'ERR_NO_RECORD_FOUND'))
}

try {
// Insert it into the peer id, in order to be validated by IPNS validator
peerId.pubKey = crypto.keys.unmarshalPublicKey(pubKey)
} catch (err) {
const errMsg = `found public key record that we couldn't convert to a value`

log.error(errMsg)
return callback(errcode(new Error(errMsg), 'ERR_INVALID_PUB_KEY_RECEIVED'))
}

this._validateRecord(peerId, ipnsEntry, callback)
})
})
}

// validate a resolved record
_validateRecord (peerId, ipnsEntry, callback) {
ipns.extractPublicKey(peerId, ipnsEntry, (err, pubKey) => {
if (err) {
return callback(err)
}

// IPNS entry validation
ipns.validate(pubKey, ipnsEntry, (err) => {
if (err) {
return callback(err)
}

callback(null, ipnsEntry.value.toString())
})
callback(null, ipnsEntry.value.toString())
})
})
}
Expand Down
13 changes: 10 additions & 3 deletions src/core/ipns/routing/utils.js
Original file line number Diff line number Diff line change
@@ -1,9 +1,16 @@
'use strict'

const multibase = require('multibase')
const ipns = require('ipns')

module.exports.encodeBase32 = (buf) => {
const m = multibase.encode('base32', buf).slice(1) // slice off multibase codec
module.exports = {
encodeBase32: (buf) => {
const m = multibase.encode('base32', buf).slice(1) // slice off multibase codec

return m.toString().toUpperCase() // should be uppercase for interop with go
return m.toString().toUpperCase() // should be uppercase for interop with go
},
validator: {
func: (key, record, cb) => ipns.validator.validate(record, key, cb)
},
selector: (k, records) => ipns.validator.select(records[0], records[1])
}
Loading