initial commit

.gitignore

@@ -0,0 +1,4 @@
+node_modules
+npm-debug.log
+.DS_Store
+esclient.log

.travis.yml

@@ -0,0 +1,13 @@
+language: node_js
+
+node_js:
+ - "0.10"
+
+install:
+ - npm install
+
+before_script:
+ - npm install npm
+
+script:
+ - npm test

mappings/partial/admin.json

@@ -0,0 +1,5 @@
+{
+  "type": "string",
+  "store": "yes",
+  "index": "no"
+}

mappings/partial/centroid.js

@@ -0,0 +1,14 @@
+
+var schema = {
+  'type': 'geo_point',
+  'lat_lon': true,
+  // 'geohash': true,
+  // 'geohash_prefix': true,
+  // 'geohash_precision': 20,
+  'fielddata' : {
+    'format' : 'compressed',
+    'precision' : '3m'
+  }
+}
+
+module.exports = schema;

mappings/partial/foreignkey.json

@@ -0,0 +1,4 @@
+{
+  "type": "long",
+  "index": "no"
+}

mappings/partial/hash.json

@@ -0,0 +1,4 @@
+{
+  "type" : "object",
+  "dynamic" : false
+}

mappings/partial/shape.json

@@ -0,0 +1,5 @@
+{
+  "type": "geo_shape",
+  "tree": "quadtree",
+  "tree_levels": "20"
+}

mappings/partial/suggest.json

@@ -0,0 +1,20 @@
+{
+  "type": "completion",
+  "preserve_position_increments": false,
+  "preserve_separators": false,
+  "index_analyzer": "suggestions",
+  "search_analyzer": "suggestions",
+  "payloads": true,
+  "context": {
+    "location": {
+      "type": "geo",
+      "precision": ["1km","100km","1500km","8000km","20040km"],
+      "neighbors": true,
+      "path": "center_point"
+    },
+    "dataset": {
+      "type": "category",
+      "path": "_type"
+    }
+  }
+}

mappings/poi-noop.js

@@ -0,0 +1,17 @@
+
+var merge = require('merge');
+
+var schema = {
+  'properties': {
+    'name':             merge( {}, require('./partial/hash'), { 'index': 'no' } ),
+    'address':          merge( {}, require('./partial/hash'), { 'index': 'no' } ),
+    'type':             merge( {}, require('./partial/admin'), { 'store': 'no', 'index': 'no' } ),
+    'admin0':           merge( {}, require('./partial/admin'), { 'index': 'no' } ),
+    'admin1':           merge( {}, require('./partial/admin'), { 'index': 'no' } ),
+    'admin2':           merge( {}, require('./partial/admin'), { 'index': 'no' } ),
+    'center_point':     { 'type': 'geo_point', 'index': 'no' },
+    'tags':             merge( {}, require('./partial/hash'), { 'index': 'no' } )
+  }
+}
+
+module.exports = schema;

mappings/poi.js

@@ -0,0 +1,18 @@
+
+var merge = require('merge');
+
+var schema = {
+  'properties': {
+    'name':             require('./partial/hash'),
+    'address':          merge( {}, require('./partial/hash'), { 'index': 'no' } ),
+    'type':             merge( {}, require('./partial/admin'), { 'store': 'no' } ),
+    'admin0':           require('./partial/admin'),
+    'admin1':           require('./partial/admin'),
+    'admin2':           require('./partial/admin'),
+    'center_point':     require('./partial/centroid'),
+    'suggest':          require('./partial/suggest'),
+    'tags':             merge( {}, require('./partial/hash'), { 'index': 'no' } )
+  }
+}
+
+module.exports = schema;

mappings/quattroshapes.js

@@ -0,0 +1,12 @@
+
+var schema = {
+  'properties': {
+    'gn_id':            require('./partial/foreignkey'),
+    'woe_id':           require('./partial/foreignkey'),
+    'boundaries':       require('./partial/shape'),
+    'center_point':     require('./partial/centroid'),
+    'suggest':          require('./partial/suggest')
+  }
+}
+
+module.exports = schema;

package.json

@@ -0,0 +1,36 @@
+{
+  "name": "pelias-schema",
+  "author": "mapzen",
+  "version": "0.0.0",
+  "description": "Elasticsearch schema files and tooling for Pelias",
+  "homepage": "https://github.com/pelias/schema",
+  "license": "MIT",
+  "main": "schema.js",
+  "scripts": {
+    "test": "node test/run.js | tap-spec"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git://github.com/pelias/schema.git"
+  },
+  "keywords": [
+    "pelias",
+    "schema"
+  ],
+  "bugs": {
+    "url": "https://github.com/pelias/schema/issues"
+  },
+  "engines": {
+    "node": ">=0.10.26",
+    "npm": ">=1.4.3",
+    "elasticsearch": ">=1.1.1"
+  },
+  "dependencies": {
+    "merge": "^1.1.3"
+  },
+  "devDependencies": {
+    "tape": "^2.13.4",
+    "tap-spec": "^0.2.0",
+    "pelias-esclient": "0.0.20"
+  }
+}

schema.js

@@ -0,0 +1,24 @@
+
+var schema = {
+  'settings':           require('./settings'),
+  'mappings': {
+
+    // osm
+    'osmnode':          require('./mappings/poi'),
+    'osmway':           require('./mappings/poi'),
+    'osmpoint':         require('./mappings/poi-noop'),
+
+    // geoname
+    'geoname':          require('./mappings/poi'),
+
+    // quattroshapes
+    'admin0':           require('./mappings/quattroshapes'),
+    'admin1':           require('./mappings/quattroshapes'),
+    'admin2':           require('./mappings/quattroshapes'),
+    'local_admin':      require('./mappings/quattroshapes'),
+    'locality':         require('./mappings/quattroshapes'),
+    'neighborhood':     require('./mappings/quattroshapes')
+  }
+}
+
+module.exports = schema;

scripts/create_index.js

@@ -0,0 +1,8 @@
+
+var client = require('pelias-esclient')(),
+    schema = require('../schema');
+
+client.indices.create( { index: 'pelias', body: schema }, function( err, res ){
+  console.log( '[put mapping]', '\t', 'pelias', err || '\t', res );
+  process.exit( !!err );
+});

scripts/drop_index.js

@@ -0,0 +1,16 @@
+
+var readline = require('readline'),
+    rl = readline.createInterface({ input: process.stdin, output: process.stdout }),
+    client = require('pelias-esclient')(),
+    schema = require('../schema');
+
+rl.question( 'Are you sure you want to drop the pelias index and delete ALL records? ', function( answer ){
+  if( !answer.match(/^y(es)?$/i) ){
+    console.log( 'you must answer \'y\' to confirm. aborting.' )
+    process.exit(0);
+  }
+  client.indices.delete( { index: 'pelias' }, function( err, res ){
+    console.log( '\n[delete mapping]', '\t', 'pelias', err || '\t', res );
+    process.exit( !!err );
+  });
+});

scripts/output_mapping.js

@@ -0,0 +1,19 @@
+
+var client = require('pelias-esclient')(),
+    schema = require('../schema');
+
+var _index = ( process.argv.length > 3 ) ? process.argv[3] : 'pelias';
+var _type = ( process.argv.length > 2 ) ? process.argv[2] : null; // get type from cli args
+
+if( !_type ){
+  console.log( JSON.stringify( schema, null, 2 ) );
+  process.exit(0);
+}
+
+var mapping = schema.mappings[_type];
+if( !mapping ){
+  console.error( 'could not find a mapping in the schema file for', _index+'/'+_type );
+  process.exit(1);
+}
+
+console.log( JSON.stringify( mapping, null, 2 ) );

scripts/reset_type.js

@@ -0,0 +1,25 @@
+
+var client = require('pelias-esclient')(),
+    schema = require('../schema');
+
+var _index = ( process.argv.length > 3 ) ? process.argv[3] : 'pelias';
+var _type = ( process.argv.length > 2 ) ? process.argv[2] : null; // get type from cli args
+
+if( !_type ){
+  console.error( 'you must provide the target es \'type\' as the first cli argument' );
+  process.exit(1);
+}
+
+var mapping = schema.mappings[_type];
+if( !mapping ){
+  console.error( 'could not find a mapping in the schema file for', _index+'/'+_type );
+  process.exit(1);
+}
+
+client.indices.deleteMapping( { index: _index, type: _type }, function( err, res ){
+  console.log( '[delete mapping]', '\t', _index+'/'+_type, err || '\t', res );
+  client.indices.putMapping( { index: _index, type: _type, body:mapping }, function( err, res ){
+    console.log( '[put mapping]', '\t\t', _index+'/'+_type, err || '\t', res );
+    process.exit( !!err );
+  });
+});

settings.json

@@ -0,0 +1,16 @@
+{
+  "analysis": {
+    "analyzer": {
+      "suggestions": {
+        "type": "custom",
+        "tokenizer": "whitespace",
+        "filter": "lowercase"
+      }
+    }
+  },
+  "index": {
+    "number_of_replicas": "0",
+    "number_of_shards": "8",
+    "index.index_concurrency": "32"
+  }
+}

test/compile.js

@@ -0,0 +1,23 @@
+
+var schema = require('../');
+
+module.exports.tests = {};
+
+module.exports.tests.compile = function(test, common) {
+  test('valid schema file', function(t) {
+    t.equal(typeof schema, 'object', 'schema generated');
+    t.equal(Object.keys(schema).length>0, true, 'schema has body');
+    t.end();
+  });
+}
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape('compile: ' + name, testFunction)
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+}

test/fixtures/_index.js

@@ -0,0 +1,9 @@
+module.exports = {
+  fields:         require('./fieldsQuery'),
+  get:            require('./getQuery'),
+  mget:           require('./mgetQuery'),
+  failedmget:     require('./mgetFailedQuery'),
+  put:            require('./putQuery'),
+  search:         require('./searchQuery'),
+  genericfail:    require('./genericFailure')
+}

test/fixtures/fieldsQuery.json

@@ -0,0 +1,35 @@
+{
+  "took": 4,
+  "timed_out": false,
+  "_shards": {
+    "total": 8,
+    "successful": 8,
+    "failed": 0
+  },
+  "hits": {
+    "total": 3589,
+    "max_score": null,
+    "hits": [
+      {
+        "_index": "pelias",
+        "_type": "geoname",
+        "_id": "6289144",
+        "_score": null,
+        "fields": {
+          "admin2": [
+            "Greater London"
+          ],
+          "admin1": [
+            "England"
+          ],
+          "admin0": [
+            "United Kingdom"
+          ]
+        },
+        "sort": [
+          0.05638587293312542
+        ]
+      }
+    ]
+  }
+}

test/fixtures/genericFailure.json

@@ -0,0 +1,4 @@
+{
+  "error": "MapperParsingException[failed to parse]; nested: JsonParseException[Unexpected character ('}' (code 125)): was expecting either valid name character (for unquoted name) or double-quote (for quoted) to start field name\n at [Source: [B@38d0af53; line: 4, column: 2]]; ",
+  "status": 400
+}

test/fixtures/getQuery.json

@@ -0,0 +1,20 @@
+{
+  "_index": "pelias",
+  "_type": "osmnode",
+  "_id": "551215562",
+  "_version": 2,
+  "found": true,
+  "_source": {
+    "center_point": {
+      "lat": 33.5169579,
+      "lon": 36.2217176
+    },
+    "name": {
+
+    },
+    "type": "node",
+    "tags": {
+
+    }
+  }
+}