From 1de0d2b8d1bc9ecce3aa151e7205f2e89b4d4ff2 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 20 Oct 2020 22:07:05 +0200 Subject: [PATCH] feat: updated dataset (2020-10-13) (#80) Co-authored-by: Irakli Gozalishvili --- README.md | 13 +- package-lock.json | 360 +++++++++++++++++++++--------------------- package.json | 4 +- src/format.js | 2 - src/generate/index.js | 70 ++++---- src/lookup.js | 12 +- test/format.spec.js | 12 +- test/generate.spec.js | 175 ++++++++++++++------ test/lookup.spec.js | 22 ++- 9 files changed, 372 insertions(+), 298 deletions(-) diff --git a/README.md b/README.md index ddd8145..8ce450b 100644 --- a/README.md +++ b/README.md @@ -70,8 +70,6 @@ Returns a promise that resolves to an object of the form "postal_code": "94040", "latitude": 37.3860, "longitude": -122.0838, - "metro_code": "807", - "area_code": "650", "planet": "Earth" } ``` @@ -85,6 +83,11 @@ a `formatted` property that looks like this: `Mountain View, CA, United States, The utility geoip-gen reads csv files provided from GeoLite, and turns them into a 32-way branching b-tree, which is stored as ipfs json objects. +**Note:** this library uses old type of ipfs json objects for legacy reasons, +be mindful of that and do not use its code as an example. Modern code should +use [`dag-cbor`](https://github.com/ipld/specs/blob/master/block-layer/codecs/dag-cbor.md) +and [`ipfs.dag`](https://github.com/ipfs/js-ipfs/blob/master/docs/core-api/DAG.md) API. + There is a generator included, that can be run with ```bash @@ -107,8 +110,6 @@ Result: { "postal_code": "94040", "latitude": 37.386, "longitude": -122.0838, - "metro_code": "807", - "area_code": "650", "planet": "Earth" } Pretty result: Mountain View, CA, USA, Earth @@ -130,4 +131,6 @@ This repository falls under the IPFS [Code of Conduct](https://github.com/ipfs/c ## License -[MIT](LICENSE) +ipfs-geoip is [MIT](LICENSE) licensed. + +This library includes GeoLite2 data created by MaxMind, available from [maxmind.com](http://www.maxmind.com). diff --git a/package-lock.json b/package-lock.json index 8c49c59..cfd46d1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "ipfs-geoip", - "version": "5.0.0", + "version": "5.0.1", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -2605,9 +2605,9 @@ } }, "@sinonjs/samsam": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-5.1.0.tgz", - "integrity": "sha512-42nyaQOVunX5Pm6GRJobmzbS7iLI+fhERITnETXzzwDZh+TtDr/Au3yAvXVjFmZ4wEUaE4Y3NFZfKv0bV0cbtg==", + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-5.2.0.tgz", + "integrity": "sha512-CaIcyX5cDsjcW/ab7HposFWzV1kC++4HNsfnEdFJa7cP1QIuILAKV+BgfeqRXhcnSAc76r/Rh/O5C+300BwUIw==", "dev": true, "requires": { "@sinonjs/commons": "^1.6.0", @@ -4172,15 +4172,15 @@ "dev": true }, "bip174": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/bip174/-/bip174-1.0.1.tgz", - "integrity": "sha512-Mq2aFs1TdMfxBpYPg7uzjhsiXbAtoVq44TNjEWtvuZBiBgc3m7+n55orYMtTAxdg7jWbL4DtH0MKocJER4xERQ==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/bip174/-/bip174-2.0.1.tgz", + "integrity": "sha512-i3X26uKJOkDTAalYAp0Er+qGMDhrbbh2o93/xiPyAN2s25KrClSpe3VXo/7mNJoqA5qfko8rLS2l3RWZgYmjKQ==", "dev": true }, "bip32": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/bip32/-/bip32-2.0.5.tgz", - "integrity": "sha512-zVY4VvJV+b2fS0/dcap/5XLlpqtgwyN8oRkuGgAS1uLOeEp0Yo6Tw2yUTozTtlrMJO3G8n4g/KX/XGFHW6Pq3g==", + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/bip32/-/bip32-2.0.6.tgz", + "integrity": "sha512-HpV5OMLLGTjSVblmrtYRfFFKuQB+GArM0+XP8HGWfJ5vxYBqo+DesvJwOdC2WJ3bCkZShGf0QIfoIpeomVzVdA==", "dev": true, "requires": { "@types/node": "10.12.18", @@ -4216,13 +4216,13 @@ "dev": true }, "bitcoinjs-lib": { - "version": "5.1.10", - "resolved": "https://registry.npmjs.org/bitcoinjs-lib/-/bitcoinjs-lib-5.1.10.tgz", - "integrity": "sha512-CesUqtBtnYc+SOMsYN9jWQWhdohW1MpklUkF7Ukn4HiAyN6yxykG+cIJogfRt6x5xcgH87K1Q+Mnoe/B+du1Iw==", + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/bitcoinjs-lib/-/bitcoinjs-lib-5.2.0.tgz", + "integrity": "sha512-5DcLxGUDejgNBYcieMIUfjORtUeNWl828VWLHJGVKZCb4zIS1oOySTUr0LGmcqJBQgTBz3bGbRQla4FgrdQEIQ==", "dev": true, "requires": { "bech32": "^1.1.2", - "bip174": "^1.0.1", + "bip174": "^2.0.1", "bip32": "^2.0.4", "bip66": "^1.1.0", "bitcoin-ops": "^1.4.0", @@ -7123,12 +7123,12 @@ }, "dependencies": { "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "dev": true, "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "ms": { @@ -7185,12 +7185,12 @@ }, "dependencies": { "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "dev": true, "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "ms": { @@ -10008,9 +10008,9 @@ "dev": true }, "fast-redact": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-2.0.0.tgz", - "integrity": "sha512-zxpkULI9W9MNTK2sJ3BpPQrTEXFNESd2X6O1tXMFpK/XM0G5c5Rll2EVYZH2TqI3xRGK/VaJ+eEOt7pnENJpeA==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-2.1.0.tgz", + "integrity": "sha512-0LkHpTLyadJavq9sRzzyqIoMZemWli77K2/MGOkafrR64B9ItrvZ9aT+jluvNDsv0YEHjSNhlMBtbokuoqii4A==", "dev": true }, "fast-safe-stringify": { @@ -12791,11 +12791,6 @@ "integrity": "sha1-gtwzbSMrkGIXnQWrMpOmYFn9Q10=", "dev": true }, - "inet_ipv4": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/inet_ipv4/-/inet_ipv4-1.0.1.tgz", - "integrity": "sha512-tn6ZkEDeiQWpgSx3Ne901r6I0GtCQNlJlhZOALo3PPvGWh7d27+lIiTCrG4g7xsKT/tQdFk8bSC8HU+ftGIpQQ==" - }, "infer-owner": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/infer-owner/-/infer-owner-1.0.4.tgz", @@ -12888,13 +12883,12 @@ "ip": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz", - "integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo=", - "dev": true + "integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo=" }, "ip-address": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-6.3.0.tgz", - "integrity": "sha512-tYN6DnF82jBRA6ZT3C+k4LBtVUKu0Taq7GZN4yldhz6nFKVh3EDg/zRIABsu4fAT2N0iFW9D482Aqkiah1NxTg==", + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-6.4.0.tgz", + "integrity": "sha512-c5uxc2WUTuRBVHT/6r4m7HIr/DfV0bF6DvLH3iZGSK8wp8iMwwZSgIq2do0asFf8q9ECug0SE+6+1ACMe4sorA==", "dev": true, "requires": { "jsbn": "1.1.0", @@ -13058,12 +13052,12 @@ } }, "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "dev": true, "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "ipfs-utils": { @@ -13169,12 +13163,12 @@ } }, "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "dev": true, "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "ms": { @@ -13480,12 +13474,12 @@ } }, "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "dev": true, "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "file-type": { @@ -13588,12 +13582,12 @@ } }, "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "dev": true, "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "is-plain-obj": { @@ -13640,9 +13634,9 @@ } }, "sort-keys": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/sort-keys/-/sort-keys-4.0.0.tgz", - "integrity": "sha512-hlJLzrn/VN49uyNkZ8+9b+0q9DjmmYcYOnbMQtpkLrYpPwRApDPZfmqbUfJnAA3sb/nRib+nDot7Zi/1ER1fuA==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/sort-keys/-/sort-keys-4.1.0.tgz", + "integrity": "sha512-/sRdxzkkPFUYiCrTr/2t+104nDc9AgDmEpeVYuvOWYQe3Djk1GWO6lVw3Vx2jfh1SsR0eehhd1nvFYlzt5e99w==", "dev": true, "requires": { "is-plain-obj": "^2.0.0" @@ -13681,12 +13675,12 @@ } }, "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "dev": true, "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "ms": { @@ -14895,12 +14889,12 @@ }, "dependencies": { "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "dev": true, "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "ms": { @@ -15746,9 +15740,9 @@ "dev": true }, "it-batch": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/it-batch/-/it-batch-1.0.4.tgz", - "integrity": "sha512-hZ+gaj5MaECauRd+Ahvo9iAxg90YGVBg7AZ32wOeXJ08IRjfQRMSnZ9oA0JjNeJeSGuVjWf91UUD5y2SYmKlwQ==", + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/it-batch/-/it-batch-1.0.6.tgz", + "integrity": "sha512-W17vOT8tZTaPMw2NcXItBIAglBz3JxNdXE0+zgqPGMk6zrEb5YF+sAn+PuNed7R6Fsnp8IKDr1sa76GL8Cp52g==", "dev": true }, "it-buffer": { @@ -15770,15 +15764,15 @@ } }, "it-drain": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/it-drain/-/it-drain-1.0.1.tgz", - "integrity": "sha512-4aX8AsJWjRh0inNXGLa90fvxuB7vQY70WFasvskUMtpXXz8+MUH8R7PODBtn4yXCJ25ud2iRwWwa1g8DRDbrlA==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/it-drain/-/it-drain-1.0.3.tgz", + "integrity": "sha512-KxwHBEpWW+0/EkGCOPR2MaHanvBW2A76tOC5CiitoJGLd8J56FxM6jJX3uow20v5qMidX5lnKgwH5oCIyYDszQ==", "dev": true }, "it-first": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/it-first/-/it-first-1.0.2.tgz", - "integrity": "sha512-hU5ObR14987PR7l0J7dfWAgKYiWoKbXcoXKqhQDGgHSZML6UPmHSS9ILBGucZkoA2B152kEqEOllS4tVQq11fg==", + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/it-first/-/it-first-1.0.4.tgz", + "integrity": "sha512-L5ZB5k3Ol5ouAzLHo6fOCtByOy2lNjteNJpZLkE+VgmRt0MbC1ibmBW1AbOt6WzDx/QXFG5C8EEvY2nTXHg+Hw==", "dev": true }, "it-glob": { @@ -15857,9 +15851,9 @@ "dev": true }, "it-multipart": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/it-multipart/-/it-multipart-1.0.3.tgz", - "integrity": "sha512-8PSjOl5OSx2fCbBJ73uV4ZVMY0Q/yCQrWHNk6XYXYDwByH5rnSBYEyuSSiOM2grDLY39atybbKTbMi3GWEbACA==", + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/it-multipart/-/it-multipart-1.0.5.tgz", + "integrity": "sha512-HW0/ycdwqM1Xz1cwkBUwmU2HTxrJrUdVZBIgX5/fNzEjIgbnL3oZUysG2NeKNbIA0vt4wnqLK6fAps/nvQ0AbA==", "dev": true, "requires": { "buffer": "^5.5.0", @@ -15877,23 +15871,23 @@ } }, "it-parallel-batch": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/it-parallel-batch/-/it-parallel-batch-1.0.4.tgz", - "integrity": "sha512-YyIa0urQO7C/YmWaKAXILv7glvvsfM9jsL+u1CUQxyO8vslLyv9i3LT8AFC55Y9r6xT3A4jK9FhaXND2NmcPFw==", + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/it-parallel-batch/-/it-parallel-batch-1.0.6.tgz", + "integrity": "sha512-ym2o1bZHZAl2euR79ojKsvVJt77DGQrmSTgDf+g3ERF/Agp2+VI9VM3ikQ9T1BBdgbSIylPeatNGMIyZgz7J7g==", "dev": true, "requires": { - "it-batch": "^1.0.4" + "it-batch": "^1.0.6" } }, "it-pb-rpc": { - "version": "0.1.8", - "resolved": "https://registry.npmjs.org/it-pb-rpc/-/it-pb-rpc-0.1.8.tgz", - "integrity": "sha512-YePzUUonithCTIdVKcOeQEn5mpipCg7ZoWsq7jfjXXtAS6gm6R7KzCe6YBV97i6bljU8hGllTG67FiGfweKNKg==", + "version": "0.1.9", + "resolved": "https://registry.npmjs.org/it-pb-rpc/-/it-pb-rpc-0.1.9.tgz", + "integrity": "sha512-IMPXz+a+lUEclV5qIlT/1WAjCMIZyqQtMRaKaL8cwgvH2P5LtMJlrbNZr3b4VEONK1H6mqAV1upfMTSSBSrOqA==", "dev": true, "requires": { "is-buffer": "^2.0.4", - "it-handshake": "^1.0.1", - "it-length-prefixed": "^3.0.1" + "it-handshake": "^1.0.2", + "it-length-prefixed": "^3.1.0" }, "dependencies": { "is-buffer": { @@ -16372,9 +16366,9 @@ "dev": true }, "just-extend": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.1.0.tgz", - "integrity": "sha512-ApcjaOdVTJ7y4r08xI5wIqpvwS48Q0PBG4DJROcEkH1f8MdAiNFyFxz3xoL0LWAVwjrwPYZdVHHxhRHcx/uGLA==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.1.1.tgz", + "integrity": "sha512-aWgeGFW67BP3e5181Ep1Fv2v8z//iBJfrvyTnq8wG86vEESwmonn1zPBJ0VfmT9CJq2FIT0VsETtrNFm2a+SHA==", "dev": true }, "just-safe-get": { @@ -16997,12 +16991,12 @@ }, "dependencies": { "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "dev": true, "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "ms": { @@ -17032,12 +17026,12 @@ }, "dependencies": { "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "dev": true, "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "ms": { @@ -17131,12 +17125,12 @@ }, "dependencies": { "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "dev": true, "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "ms": { @@ -17165,12 +17159,12 @@ }, "dependencies": { "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "dev": true, "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "ms": { @@ -17200,12 +17194,12 @@ }, "dependencies": { "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "dev": true, "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "ms": { @@ -17235,12 +17229,12 @@ }, "dependencies": { "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "dev": true, "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "ms": { @@ -17339,12 +17333,12 @@ } }, "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "dev": true, "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "ms": { @@ -17428,12 +17422,12 @@ }, "dependencies": { "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "dev": true, "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "ms": { @@ -17461,12 +17455,12 @@ }, "dependencies": { "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "dev": true, "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "ms": { @@ -17497,12 +17491,12 @@ }, "dependencies": { "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "dev": true, "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "ms": { @@ -17532,12 +17526,12 @@ }, "dependencies": { "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "dev": true, "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "ms": { @@ -17652,12 +17646,12 @@ }, "dependencies": { "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "dev": true, "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "libp2p-interfaces": { @@ -17754,12 +17748,12 @@ }, "dependencies": { "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "dev": true, "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "ms": { @@ -17784,12 +17778,12 @@ }, "dependencies": { "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "dev": true, "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "ms": { @@ -17815,12 +17809,12 @@ }, "dependencies": { "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "dev": true, "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "ms": { @@ -17862,12 +17856,12 @@ }, "dependencies": { "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "dev": true, "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "ms": { @@ -17910,12 +17904,12 @@ }, "dependencies": { "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "dev": true, "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "ms": { @@ -20237,12 +20231,12 @@ }, "dependencies": { "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "dev": true, "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "ms": { @@ -20408,15 +20402,15 @@ } }, "node-fetch": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", - "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==", + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==", "dev": true }, "node-forge": { - "version": "0.9.1", - "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.9.1.tgz", - "integrity": "sha512-G6RlQt5Sb4GMBzXvhfkeFmbqR6MzhtnT7VTHuLadjkii3rdYHNdw0m8zA4BTxVIh68FicCQ2NSUANpsqkr9jvQ==", + "version": "0.9.2", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.9.2.tgz", + "integrity": "sha512-naKSScof4Wn+aoHU6HBsifh92Zeicm1GDQKd1vp3Y/kOi8ub0DozCa9KpvYNCXslFHYRmLNiqRopGdTGwNLpNw==", "dev": true }, "node-gyp-build": { @@ -21242,13 +21236,13 @@ } }, "p-queue": { - "version": "6.6.1", - "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.1.tgz", - "integrity": "sha512-miQiSxLYPYBxGkrldecZC18OTLjdUqnlRebGzPRiVxB8mco7usCmm7hFuxiTvp93K18JnLtE4KMMycjAu/cQQg==", + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", + "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", "dev": true, "requires": { "eventemitter3": "^4.0.4", - "p-timeout": "^3.1.0" + "p-timeout": "^3.2.0" }, "dependencies": { "p-timeout": { @@ -22120,9 +22114,9 @@ }, "dependencies": { "@types/node": { - "version": "10.17.28", - "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.28.tgz", - "integrity": "sha512-dzjES1Egb4c1a89C7lKwQh8pwjYmlOAG9dW1pBgxEk57tMrLnssOfEthz8kdkNaBd7lIqQx7APm5+mZ619IiCQ==", + "version": "10.17.40", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.40.tgz", + "integrity": "sha512-3hZT2z2/531A5pc8hYhn1gU5Qb1SIRSgMLQ6zuHA5xtt16lWAxUGprtr8lJuc9zNJMXEIIBWfSnzqBP/4mglpA==", "dev": true } } @@ -22326,12 +22320,12 @@ }, "dependencies": { "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "dev": true, "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "ms": { @@ -23437,15 +23431,15 @@ } }, "sinon": { - "version": "9.0.3", - "resolved": "https://registry.npmjs.org/sinon/-/sinon-9.0.3.tgz", - "integrity": "sha512-IKo9MIM111+smz9JGwLmw5U1075n1YXeAq8YeSFlndCLhAL5KGn6bLgu7b/4AYHTV/LcEMcRm2wU2YiL55/6Pg==", + "version": "9.2.0", + "resolved": "https://registry.npmjs.org/sinon/-/sinon-9.2.0.tgz", + "integrity": "sha512-eSNXz1XMcGEMHw08NJXSyTHIu6qTCOiN8x9ODACmZpNQpr0aXTBXBnI4xTzQzR+TEpOmLiKowGf9flCuKIzsbw==", "dev": true, "requires": { - "@sinonjs/commons": "^1.7.2", + "@sinonjs/commons": "^1.8.1", "@sinonjs/fake-timers": "^6.0.1", "@sinonjs/formatio": "^5.0.1", - "@sinonjs/samsam": "^5.1.0", + "@sinonjs/samsam": "^5.2.0", "diff": "^4.0.2", "nise": "^4.0.4", "supports-color": "^7.1.0" @@ -24262,9 +24256,9 @@ } }, "streaming-iterables": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/streaming-iterables/-/streaming-iterables-5.0.2.tgz", - "integrity": "sha512-9z5iBWe9WXzdT0X1JT9fVC0mCcVxWt5yzZMBUIgjZnt2k23+UQF8Ac6kiI8DnlYZJn5iysvxKl3uGzlijMQ+/g==", + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/streaming-iterables/-/streaming-iterables-5.0.3.tgz", + "integrity": "sha512-1AgrKjHTvaaK+iA+N3BuTXQWVb7Adyb6+v8yIW3SCTwlBVYEbm76mF8Mf0/IVo+DOk7hoeELOURBKTCMhe/qow==", "dev": true }, "streamroller": { diff --git a/package.json b/package.json index 9270417..8bd57dd 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ }, "scripts": { "lint": "aegir lint", - "release": "aegir release", + "release": "aegir release --node", "build": "aegir build", "test": "aegir test --node", "test:node": "aegir test --node --target node", @@ -32,7 +32,7 @@ }, "dependencies": { "cids": "^1.0.0", - "inet_ipv4": "^1.0.1", + "ip": "^1.1.5", "it-concat": "^1.0.0", "p-memoize": "^4.0.0" }, diff --git a/src/format.js b/src/format.js index 00513f3..8dbde0e 100644 --- a/src/format.js +++ b/src/format.js @@ -12,8 +12,6 @@ module.exports = function formatData (data) { postal_code: data[4], latitude: data[5], longitude: data[6], - metro_code: data[7], - area_code: data[8], planet: PLANET } } diff --git a/src/generate/index.js b/src/generate/index.js index 759cfaf..6ad5760 100644 --- a/src/generate/index.js +++ b/src/generate/index.js @@ -3,6 +3,7 @@ const Promise = require('bluebird') const csv = Promise.promisifyAll(require('csv')) const iconv = require('iconv-lite') +const ip = require('ip') const _ = require('lodash') const EventEmitter = require('events').EventEmitter const concat = require('it-concat') @@ -12,14 +13,15 @@ const normalizeName = require('./overrides') // Btree size const CHILDREN = 32 -// All data is stored in an ipfs folder called data -// this is the hash of that folder. It includes three files +// All data is stored in an ipfs folder called DATA_HASH +// It includes two files // -// data -// |- blocks.csv -// |- countries.csv -// |- locations.csv -const DATA_HASH = 'QmTMh5Q1CnB9jV774aKCvPSqibwDy9sJmo7BCThD5f1oY3' +// DATA_HASH +// |- locationsCsv +// |- blocksCsv +const DATA_HASH = 'bafybeid3munsqqt36qhoumn3kvgwmft6dsswzgl3wiohsanlyqemczcsvi' // GeoLite2-City-CSV_20201013 +const locationsCsv = 'GeoLite2-City-Locations-en.csv' +const blocksCsv = 'GeoLite2-City-Blocks-IPv4.csv' const progress = new EventEmitter() @@ -30,16 +32,18 @@ function emit (type, status, attrs) { }, attrs)) } -function parseCountries (countries) { +function parseCountries (locations) { emit('countries', 'start') - return csv.parseAsync(countries, { + return csv.parseAsync(locations, { columns: true, cast: false, skip_empty_lines: true }) .then((parsed) => { return _.reduce(parsed, (acc, row) => { - acc[row.alpha2] = normalizeName(row.name) + if (typeof acc[row.country_iso_code] != null) { // eslint-disable-line valid-typeof + acc[row.country_iso_code] = normalizeName(row.country_name) + } return acc }, {}) }) @@ -59,16 +63,11 @@ function parseLocations (locations, countries) { }) .then((parsed) => { return _.reduce(parsed, (acc, row) => { - acc[row.locId] = [ - countries[row.country], - row.country, - row.region, - normalizeName(row.city), - row.postalCode, - Number(row.latitude), - Number(row.longitude), - row.metroCode, - row.areaCode + acc[row.geoname_id] = [ + countries[row.country_iso_code], + row.country_iso_code, + row.subdivision_1_iso_code, + normalizeName(row.city_name) ] return acc }, {}) @@ -88,12 +87,25 @@ function parseBlocks (blocks, locations) { comment: '#' }) .then((parsed) => { - var lastEnd = 0 + let lastEnd = 0 return _.reduce(parsed, (acc, row) => { - var start = Number(row.startIpNum) - var end = Number(row.endIpNum) - var locid = row.locId + const { firstAddress, lastAddress } = ip.cidrSubnet(row.network) + const start = ip.toLong(firstAddress) + const end = ip.toLong(lastAddress) + + const { geoname_id } = row // eslint-disable-line camelcase + const geonameData = locations[geoname_id] + + // conform to legacy input format by filling up missing data the first time + // a geoname is inspected + if (Array.isArray(geonameData) && geonameData.length < 7) { + geonameData.push( + String(row.postal_code), + Number(row.latitude), + Number(row.longitude) + ) + } // unmapped range? if ((start - lastEnd) > 1) { @@ -105,7 +117,7 @@ function parseBlocks (blocks, locations) { acc.push({ min: start, - data: locations[locid] + data: geonameData }) lastEnd = end @@ -139,6 +151,7 @@ function putObject (data, min, api) { // Create a btree leaf with data function createLeaf (data) { + // TODO: use dag-cbor instead of stringified JSON return Buffer.from(JSON.stringify({ Data: JSON.stringify({ type: 'Leaf', @@ -148,6 +161,7 @@ function createLeaf (data) { } // Create a btree node with data function createNode (data) { + // TODO: use dag-cbor instead of stringified JSON return Buffer.from(JSON.stringify({ Data: JSON.stringify({ type: 'Node', @@ -189,15 +203,15 @@ async function file (ipfs, dir) { } function main (ipfs) { - return file(ipfs, 'countries.csv') + return file(ipfs, locationsCsv) .then(parseCountries) .then((countries) => Promise.join( - file(ipfs, 'locations.csv'), + file(ipfs, locationsCsv), countries, parseLocations )) .then((locations) => Promise.join( - file(ipfs, 'blocks.csv'), + file(ipfs, blocksCsv), locations, parseBlocks )) diff --git a/src/lookup.js b/src/lookup.js index 383e2fa..24a53df 100644 --- a/src/lookup.js +++ b/src/lookup.js @@ -1,12 +1,12 @@ 'use strict' const memoize = require('p-memoize') -const inet = require('inet_ipv4') +const ip = require('ip') const CID = require('cids') const formatData = require('./format') -const GEOIP_ROOT = new CID('QmPeUAhjacm8v59z5rZhCEhvWvwvA2DoMVY2w7ZuwbQGjv') +const GEOIP_ROOT = new CID('Qmbt1YbZAhMoqo7r1t6Y5EJrYGVRgcaisNAZhLeJY6ehfg') // GeoLite2-City-CSV_20201013 /** * @param {Object} ipfs @@ -19,6 +19,8 @@ async function _lookup (ipfs, cid, lookfor) { // to avoid serialization issues when mix of cids >1.0 and <1.0 are used cid = new CID(cid).toString() const res = await ipfs.object.get(cid) + + // TODO: use dag-cbor instead of stringified JSON const obj = JSON.parse(res.Data) let child = 0 @@ -71,11 +73,11 @@ const memoizedLookup = memoize(_lookup, { /** * @param {Object} ipfs - * @param {string} ip + * @param {string} ipstring * @returns {Promise} */ -module.exports = function lookup (ipfs, ip) { - return memoizedLookup(ipfs, GEOIP_ROOT, inet.aton(ip)) +module.exports = function lookup (ipfs, ipstring) { + return memoizedLookup(ipfs, GEOIP_ROOT, ip.toLong(ipstring)) } function getCid (node) { diff --git a/test/format.spec.js b/test/format.spec.js index 3e1685f..197bc7b 100644 --- a/test/format.spec.js +++ b/test/format.spec.js @@ -15,9 +15,7 @@ describe('format', () => { 'Mountain View', '94040', 37.386, - -122.0838, - '807', - '650' + -122.0838 ]) ).to.be.eql({ country_name: 'USA', @@ -27,8 +25,6 @@ describe('format', () => { postal_code: '94040', latitude: 37.386, longitude: -122.0838, - metro_code: '807', - area_code: '650', planet: 'Earth' }) }) @@ -42,9 +38,7 @@ describe('format', () => { '', '', 37.386, - -122.0838, - '', - '' + -122.0838 ]) ).to.be.eql({ country_name: 'USA', @@ -54,8 +48,6 @@ describe('format', () => { postal_code: '', latitude: 37.386, longitude: -122.0838, - metro_code: '', - area_code: '', planet: 'Earth' }) }) diff --git a/test/generate.spec.js b/test/generate.spec.js index 3679f48..de2e6d1 100644 --- a/test/generate.spec.js +++ b/test/generate.spec.js @@ -11,31 +11,24 @@ const expect = chai.expect const gen = require('../src/generate/') -const countries = Buffer.from(` -name,alpha2,countryCallingCodes,alpha3,ioc,currencies,languages,ccTLD,status -Ascension Island,AC,+247,,SHP,USD,eng,.ac,reserved -Andorra,AD,+376,AND,AND,EUR,cat,,assigned -United Arab Emirates,AE,+971,ARE,UAE,AED,ara,,assigned -Afghanistan,AF,+93,AFG,AFG,AFN,pus,,assigned -Antigua And Barbuda,AG,+1 268,ATG,ANT,XCD,eng,,assigned -`) - const locations = Buffer.from(` -# Copyright (c) 2012 MaxMind LLC. All Rights Reserved. -locId,country,region,city,postalCode,latitude,longitude,metroCode,areaCode -1,"AD","","","",42.5000,1.5000,, -2,"AE","","","",24.0000,54.0000,, -3,"AF","","","",33.0000,65.0000,, -4,"AG","","","",17.0500,-61.8000,, +geoname_id,locale_code,continent_code,continent_name,country_iso_code,country_name,subdivision_1_iso_code,subdivision_1_name,subdivision_2_iso_code,subdivision_2_name,city_name,metro_code,time_zone,is_in_european_union +3039163,en,EU,Europe,AD,Andorra,06,"Sant Julià de Loria",,,"Sant Julià de Lòria",,Europe/Andorra,0 +12042053,en,AS,Asia,AE,"United Arab Emirates",AZ,"Abu Dhabi",,,"Musaffah City",,Asia/Dubai,0 +765876,en,EU,Europe,PL,Poland,06,Lublin,,,Lublin,,Europe/Warsaw,1 +5391959,en,NA,"North America",US,"United States",CA,California,,,"San Francisco",807,America/Los_Angeles,0 `) +// the old format had separate files, +// however now we read data from the same CSV as locations +const countries = locations + const blocks = Buffer.from(` -# Copyright (c) 2011 MaxMind Inc. All Rights Reserved. -startIpNum,endIpNum,locId -"16777216","16777471","1" -"16777472","16778239","2" -"16778240","16779263","3" -"16779264","16781311","4" +network,geoname_id,registered_country_geoname_id,represented_country_geoname_id,is_anonymous_proxy,is_satellite_provider,postal_code,latitude,longitude,accuracy_radius +194.158.92.192/26,3039163,3041565,,0,0,,42.4678,1.5005,100 +94.59.56.0/24,12042053,290557,,0,0,,24.3613,54.4803,20 +213.195.159.0/24,765876,798544,,0,0,20-128,51.2574,22.5850,200 +2.56.139.0/24,5391959,2017370,,0,0,94119,37.7794,-122.4176,1000 `) const enc = new TextEncoder() @@ -52,55 +45,135 @@ describe('generate', () => { return expect( gen.parseCountries(countries) ).to.eventually.be.eql({ - AC: 'Ascension Island', AD: 'Andorra', AE: 'United Arab Emirates', - AF: 'Afghanistan', - AG: 'Antigua and Barbuda' + PL: 'Poland', + US: 'USA' }) }) it('parseLocations', () => { return expect( gen.parseLocations(locations, { - AC: 'Ascension Island', AD: 'Andorra', AE: 'United Arab Emirates', - AF: 'Afghanistan', - AG: 'Antigua and Barbuda' + PL: 'Poland', + US: 'USA' }) ).to.eventually.be.eql({ - 1: ['Andorra', 'AD', '', '', '', 42.5, 1.5, '', ''], - 2: ['United Arab Emirates', 'AE', '', '', '', 24, 54, '', ''], - 3: ['Afghanistan', 'AF', '', '', '', 33, 65, '', ''], - 4: ['Antigua and Barbuda', 'AG', '', '', '', 17.05, -61.8, '', ''] + 765876: [ + 'Poland', + 'PL', + '06', + 'Lublin' + ], + 3039163: [ + 'Andorra', + 'AD', + '06', + 'Sant Julià de Lòria' + ], + 5391959: [ + 'USA', + 'US', + 'CA', + 'San Francisco' + ], + 12042053: [ + 'United Arab Emirates', + 'AE', + 'AZ', + 'Musaffah City' + ] }) }) it('parseBlocks', () => { return expect( gen.parseBlocks(blocks, { - 1: ['Andorra', 'AD', '', '', '', 42.5, 1.5, '', ''], - 2: ['United Arab Emirates', 'AE', '', '', '', 24, 54, '', ''], - 3: ['Afghanistan', 'AF', '', '', '', 33, 65, '', ''], - 4: ['Antigua and Barbuda', 'AG', '', '', '', 17.05, -61.8, '', ''] + 765876: [ + 'Poland', + 'PL', + '06', + 'Lublin' + ], + 3039163: [ + 'Andorra', + 'AD', + '06', + 'Sant Julià de Lòria' + ], + 5391959: [ + 'USA', + 'US', + 'CA', + 'San Francisco' + ], + 12042053: [ + 'United Arab Emirates', + 'AE', + 'AZ', + 'Musaffah City' + ] }) - ).to.eventually.be.eql([{ - min: 1, - data: 0 - }, { - min: 16777216, - data: ['Andorra', 'AD', '', '', '', 42.5, 1.5, '', ''] - }, { - min: 16777472, - data: ['United Arab Emirates', 'AE', '', '', '', 24, 54, '', ''] - }, { - min: 16778240, - data: ['Afghanistan', 'AF', '', '', '', 33, 65, '', ''] - }, { - min: 16779264, - data: ['Antigua and Barbuda', 'AG', '', '', '', 17.05, -61.8, '', ''] - }]) + ).to.eventually.be.eql([ + { + data: 0, + min: 1 + }, + { + data: [ + 'Andorra', + 'AD', + '06', + 'Sant Julià de Lòria', + '', + 42.4678, + 1.5005 + ], + min: 3265158337 + }, + { + data: [ + 'United Arab Emirates', + 'AE', + 'AZ', + 'Musaffah City', + '', + 24.3613, + 54.4803 + ], + min: 1580939265 + }, + { + data: 0, + min: 1580939519 + }, + { + data: [ + 'Poland', + 'PL', + '06', + 'Lublin', + '20-128', + 51.2574, + 22.585 + ], + min: 3586367233 + }, + { + data: [ + 'USA', + 'US', + 'CA', + 'San Francisco', + '94119', + 37.7794, + -122.4176 + ], + min: 37260033 + } + ]) }) it('putObject', () => { diff --git a/test/lookup.spec.js b/test/lookup.spec.js index 95408fd..71c4632 100644 --- a/test/lookup.spec.js +++ b/test/lookup.spec.js @@ -27,20 +27,18 @@ describe('lookup', function () { } }) - it('looks up 8.8.8.8', async () => { - const result = await geoip.lookup(ipfs, '8.8.8.8') + it('looks up 66.6.44.4', async () => { + const result = await geoip.lookup(ipfs, '66.6.44.4') expect( result ).to.be.eql({ country_name: 'USA', country_code: 'US', - region_code: 'CA', - city: 'Mountain View', - postal_code: '94040', - latitude: 37.386, - longitude: -122.0838, - metro_code: '807', - area_code: '650', + region_code: 'NY', + city: 'New York', + postal_code: '10004', + latitude: 40.7126, + longitude: -74.0066, planet: 'Earth' }) }) @@ -54,11 +52,11 @@ describe('lookup', function () { } }) - it('looks up 8.8.8.8', async () => { - const result = await geoip.lookupPretty(ipfs, '/ip4/8.8.8.8') + it('looks up 66.6.44.4', async () => { + const result = await geoip.lookupPretty(ipfs, '/ip4/66.6.44.4') expect( result.formatted - ).to.be.eql('Mountain View, CA, USA, Earth') + ).to.be.eql('New York, NY, USA, Earth') }) })