Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Lookup pretty #3

Merged
merged 4 commits into from
Oct 21, 2015
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
node_modules
npm-debug.log
64 changes: 46 additions & 18 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,39 +1,67 @@
# geoip lookup over ipfs
# IPFS GeoIP

Proof of concept
> *Proof of concept:* Geoip lookup over ipfs

# b-tree

The utility geoip-gen reads csv files provided from GeoLite, and turns them into a 32-way branching b-tree, which is stored as ipfs json objects.
## API

#
### `lookup(ipfs, ip, callback)`

includes the generator, that can be called like this:
Returns an object of the form

```bash
node geoip-gen.js path/GeoLite-Blocks.csv path/GeoLite-Location.csv
```js
{
"country_code": "US",
"country_name": "United States",
"region_code": "CA",
"city": "Mountain View",
"postal_code": "94040",
"latitude": 37.3860,
"longitude": -122.0838,
"metro_code": "807",
"area_code": "650",
"planet": "Earth"
}
```

This takes quite a long time to import, but you only need to do it once globally to use the lookup feature.
### `lookupPretty(ipfs, multiaddrs, callback)`

and a lookup, for example, in the example directory
Provides the same results as `lookup` with the addition of
a `formatted` property that looks like this: `Mountain View, CA, United States, Earth`.

```
node example/lookup.js 8.8.8.8
## b-tree

The utility geoip-gen reads csv files provided from GeoLite, and turns them into a 32-way branching b-tree, which is stored as ipfs json objects.

There is a generator included, that can be called like this:

```bash
$ node geoip-gen.js path/GeoLite-Blocks.csv path/GeoLite-Location.csv
```

which will result in:
This takes quite a long time to import, but you only need to do it once globally to use the lookup feature.

```js
{
"country_code": "US",
## Example

You can find an example of how to use this in [`example/lookup.js`](example/lookup.js), which you can use like this:

```bash
$ node example/lookup.js 8.8.8.8
Result: {
"country_name": "United States",
"country_code": "US",
"region_code": "CA",
"city": "Mountain View",
"postal_code": "94040",
"latitude": 37.3860,
"latitude": 37.386,
"longitude": -122.0838,
"metro_code": "807",
"area_code": "650"
"area_code": "650",
"planet": "Earth"
}
Pretty result: Mountain View, CA, United States, Earth
```

## License

[MIT](LICENSE)
18 changes: 14 additions & 4 deletions example/lookup.js
Original file line number Diff line number Diff line change
@@ -1,15 +1,25 @@
'use strict'

var geoip = require('../')
var ipfs = require('ipfs-api')()

if (process.argv.length != 3) {
console.log("usage: node lookup.js <ip4-adr>")
if (process.argv.length !== 3) {
console.log('usage: node lookup.js <ip4-adr>')
process.exit(1)
}

geoip.lookup(ipfs, process.argv[2], function (err, result) {
if (err) {
console.log("Error: " + err)
console.log('Error: ' + err)
} else {
console.log('Result: ' + JSON.stringify(result, null, 2))
}
})

geoip.lookupPretty(ipfs, '/ip4/' + process.argv[2], function (err, result) {
if (err) {
console.log('Error: ' + err)
} else {
console.log("Result: " + JSON.stringify(result, null, 2))
console.log('Pretty result: %s', result.formatted)
}
})
1 change: 0 additions & 1 deletion generate/geoip-gen.js
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,6 @@ function toNode (things) {
data: things})

data = new Buffer(JSON.stringify({Data: leaf}))

} else {
// btree node
var node = JSON.stringify({type: 'Node',
Expand Down
73 changes: 7 additions & 66 deletions ipfs-geoip.js
Original file line number Diff line number Diff line change
@@ -1,70 +1,11 @@
'use strict'
var memoize = require('memoizee')

var GEOIP_ROOT = 'QmQQ3BUpPjgYiTdhp4H9YWSCtoFXs8t91njhpvXNNLd3yB'
var l = require('./lib/lookup')
var lookupPretty = require('./lib/pretty')

function aton4 (a) {
a = a.split(/\./)
return ((parseInt(a[0], 10) << 24) >>> 0) + ((parseInt(a[1], 10) << 16) >>> 0) + ((parseInt(a[2], 10) << 8) >>> 0) + (parseInt(a[3], 10) >>> 0)
module.exports = {
lookup: l.lookup,
lookup_root: l.lookup_root,
_lookup: l._lookup,
lookupPretty: lookupPretty
}

function formatData (data) {
var obj = {}

if (data[0]) obj.country_name = data[0]
if (data[1]) obj.country_code = data[1]
if (data[2]) obj.region_code = data[2]
if (data[3]) obj.city = data[3]
if (data[4]) obj.postal_code = data[4]
if (data[5]) obj.latitude = parseFloat(data[5])
if (data[6]) obj.longitude = parseFloat(data[6])
if (data[7]) obj.metro_code = data[7]
if (data[8]) obj.area_code = data[8]

return obj
}

var memoized_lookup

function _lookup (ipfs, hash, lookfor, cb) {
ipfs.object.get(hash, function (err, res) {
if (err) {
cb(err, null)
} else {
var obj = JSON.parse(res.Data)

var child = 0
if (obj.type === 'Node') {
while (obj.mins[child] &&
obj.mins[child] <= lookfor) {
child++
}
return memoized_lookup(ipfs, res.Links[child - 1].Hash, lookfor, cb)
} else if (obj.type === 'Leaf') {
while (obj.data[child] &&
obj.data[child].min <= lookfor) {
child++
}
if (obj.data[child - 1].data) {
cb(null, formatData(obj.data[child - 1].data))
} else {
cb('Unmapped range', null)
}
}
}
})
}

memoized_lookup = memoize(_lookup, {async: true})

function lookup (ipfs, ip, cb) {
memoized_lookup(ipfs, GEOIP_ROOT, aton4(ip), cb)
}

function lookup_root (ipfs, hash, ip, cb) {
memoized_lookup(ipfs, hash, aton4(ip), cb)
}

module.exports = {lookup: lookup,
lookup_root: lookup_root,
_lookup: memoized_lookup}
10 changes: 10 additions & 0 deletions lib/aton4.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
'use strict'

module.exports = function aton4 (a) {
a = a.split(/\./)

return ((parseInt(a[0], 10) << 24) >>> 0) +
((parseInt(a[1], 10) << 16) >>> 0) +
((parseInt(a[2], 10) << 8) >>> 0) +
(parseInt(a[3], 10) >>> 0)
}
22 changes: 22 additions & 0 deletions lib/format.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
'use strict'

// TODO(dignifiedquire): Adjust for more planets
var PLANET = 'Earth'

module.exports = function formatData (data) {
var obj = {}

if (data[0]) obj.country_name = data[0]
if (data[1]) obj.country_code = data[1]
if (data[2]) obj.region_code = data[2]
if (data[3]) obj.city = data[3]
if (data[4]) obj.postal_code = data[4]
if (data[5]) obj.latitude = parseFloat(data[5])
if (data[6]) obj.longitude = parseFloat(data[6])
if (data[7]) obj.metro_code = data[7]
if (data[8]) obj.area_code = data[8]

obj.planet = PLANET
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

:)


return obj
}
56 changes: 56 additions & 0 deletions lib/lookup.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
'use strict'

var memoize = require('memoizee')

var formatData = require('./format')
var aton4 = require('./aton4')

var GEOIP_ROOT = 'QmQQ3BUpPjgYiTdhp4H9YWSCtoFXs8t91njhpvXNNLd3yB'

var memoized_lookup

function _lookup (ipfs, hash, lookfor, cb) {
ipfs.object.get(hash, function (err, res) {
if (err) {
cb(err, null)
} else {
var obj = JSON.parse(res.Data)

var child = 0
if (obj.type === 'Node') {
while (obj.mins[child] &&
obj.mins[child] <= lookfor) {
child++
}
return memoized_lookup(ipfs, res.Links[child - 1].Hash, lookfor, cb)
} else if (obj.type === 'Leaf') {
while (obj.data[child] &&
obj.data[child].min <= lookfor) {
child++
}

if (obj.data[child - 1].data) {
cb(null, formatData(obj.data[child - 1].data))
} else {
cb('Unmapped range', null)
}
}
}
})
}

memoized_lookup = memoize(_lookup, {async: true})

function lookup (ipfs, ip, cb) {
memoized_lookup(ipfs, GEOIP_ROOT, aton4(ip), cb)
}

function lookup_root (ipfs, hash, ip, cb) {
memoized_lookup(ipfs, hash, aton4(ip), cb)
}

module.exports = {
lookup: lookup,
lookup_root: lookup_root,
_lookup: memoized_lookup
}
39 changes: 39 additions & 0 deletions lib/pretty.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
'use strict'

var lookup = require('./lookup').lookup

function isLocal (address) {
var split = address.split('.')
if (split[0] === '10') return true
if (split[0] === '127') return true
if (split[0] === '192' && split[1] === '168') return true
if (split[0] === '172' && +split[1] >= 16 && +split[1] <= 31) return true
return false
}

module.exports = function lookupPretty (ipfs, multiaddrs, cb) {
if (multiaddrs.length === 0) return cb(null, null)
if (typeof multiaddrs === 'string') multiaddrs = [multiaddrs]

var address = multiaddrs[0].split('/')[2]
if (isLocal(address)) return lookupPretty(ipfs, multiaddrs.slice(1), cb)

lookup(ipfs, address, function (err, res) {
if (err) return cb(err)

if (!res.country_name && multiaddrs.length > 1) {
return lookupPretty(ipfs, multiaddrs.slice(1), cb)
}

var location = []

if (res.planet) location.push(res.planet)
if (res.country_name) location.unshift(res.country_name)
if (res.region_code) location.unshift(res.region_code)
if (res.city) location.unshift(res.city)

res.formatted = location.join(', ')

cb(null, res)
})
}
11 changes: 6 additions & 5 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,12 @@
},
"devDependencies": {
"csv": "^0.4.2",
"kew": "^0.5.0",
"ipfs-api": "^1.2.1",
"standard": "^3.3.2",
"eslint-plugin-react": "^3.5.1",
"iconv-lite": "^0.4.8",
"pre-commit": "^1.0.6"
"ipfs-api": "^2.4.1",
"kew": "^0.7.0",
"pre-commit": "^1.0.6",
"standard": "^5.3.1"
},
"keywords": [
"ipfs",
Expand All @@ -33,7 +34,7 @@
},
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
"lint": "git diff --name-only --cached --relative | egrep .js$ | xargs --no-run-if-empty standard"
"lint": "node_modules/.bin/standard"
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i think @diasdavid used the git diff mode? not sure what the benefit is there

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nah, that gif diff madness is not mine :) and I already found times that super fast linting check (with git diff) didn't caught linting errors that lead to mem leaks in another project. I recommend running standard and let it check everything.

},
"pre-commit": [
"lint"
Expand Down