Skip to content

Commit

Permalink
add file
Browse files Browse the repository at this point in the history
  • Loading branch information
@JThink
JThink committed Aug 8, 2017
1 parent 68a5962 commit aeb304d
Show file tree
Hide file tree
Showing 8 changed files with 164 additions and 0 deletions.
6 changes: 6 additions & 0 deletions skyeye-collector/skyeye-collector-core/src/main/resources/shell/create-topic.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
#!/bin/sh

# zk servers, 192.168.88.70:2181,192.168.88.71:2181,192.168.88.72:2181/kafka/0.10.0.1
urls=$1

bin/kafka-topics.sh --create --zookeeper ${urls} --replication-factor 3 --partitions 3 --topic app-log
59 changes: 59 additions & 0 deletions skyeye-collector/skyeye-collector-indexer/src/main/resources/shell/create-index.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
#!/usr/bin/python198
# -*- coding: UTF-8 -*-

import sys
import datetime
from pyelasticsearch import ElasticSearch
from pyelasticsearch import bulk_chunks

def main(argv):
index = argv[1]
doc_type = 'log'
url = []
urls = argv[2].strip().split(',')
for u in urls:
url.append(u)

es = ElasticSearch(urls = url, timeout = 60, max_retries = 0)
create_mapping(es, index, doc_type)

def create_mapping(es, index, doc_type):
mapping = {
'settings': {
'index': {
'number_of_replicas': 1,
'number_of_shards': 6,
'refresh_interval': '5s'
}
},
'mappings': {
'_default_': {
'_all': {
'enabled': False
}
},
doc_type : {
'properties' : {
'day': { 'type': 'string', 'index': 'not_analyzed'},
'time': { 'type': 'string', 'index': 'not_analyzed'},
'nanoTime': { 'type': 'string', 'index': 'not_analyzed'},
'created': { 'type': 'date', 'index': 'not_analyzed'},
'app': { 'type': 'string', 'index': 'not_analyzed'},
'host': { 'type': 'string', 'index': 'not_analyzed'},
'thread': { 'type': 'string', 'index': 'not_analyzed'},
'level': { 'type': 'string', 'index': 'not_analyzed'},
'eventType': { 'type': 'string', 'index': 'not_analyzed'},
'pack': { 'type': 'string', 'index': 'not_analyzed'},
'clazz': { 'type': 'string', 'index': 'not_analyzed'},
'line': { 'type': 'string', 'index': 'not_analyzed'},
'messageSmart': { 'type': 'string', 'analyzer': 'ik_smart', 'search_analyzer': 'ik_smart', 'include_in_all': 'true', 'boost': 8},
'messageMax': { 'type': 'string', 'analyzer': 'ik_max_word', 'search_analyzer': 'ik_max_word', 'include_in_all': 'true', 'boost': 8}
}
}
}
}
es.create_index(index = index, settings = mapping)


if __name__ == '__main__':
main(sys.argv)
6 changes: 6 additions & 0 deletions skyeye-collector/skyeye-collector-indexer/src/main/resources/shell/install.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
#!/bin/bash

echo "--> install python-pip module"
sudo apt-get install python-pip
echo "--> install elasticsearch module"
sudo pip install pyelasticsearch
12 changes: 12 additions & 0 deletions skyeye-collector/skyeye-collector-indexer/src/main/resources/shell/start.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/bash

# 索引
# urls http://192.168.88.70:9200,http://192.168.88.71:9200,http://192.168.88.72:9200
# call: bash start.sh app-log http://192.168.88.70:9200,http://192.168.88.71:9200,http://192.168.88.72:9200

index=$1
urls=$2

echo '创建索引开始'
python create-index.py ${index} ${urls}
echo '创建索引结束'
60 changes: 60 additions & 0 deletions skyeye-collector/skyeye-collector-metrics/src/main/resources/shell/create-index.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
#!/usr/bin/python198
# -*- coding: UTF-8 -*-

import sys
import datetime
from pyelasticsearch import ElasticSearch
from pyelasticsearch import bulk_chunks

def main(argv):
index = argv[1]
doc_type = 'log'
url = []
urls = argv[2].strip().split(',')
for u in urls:
url.append(u)

es = ElasticSearch(urls = url, timeout = 60, max_retries = 0)
create_mapping(es, index, doc_type)

def create_mapping(es, index, doc_type):
mapping = {
'settings': {
'index': {
'number_of_replicas': 1,
'number_of_shards': 6,
'refresh_interval': '5s'
}
},
'mappings': {
'_default_': {
'_all': {
'enabled': False
}
},
doc_type : {
'properties' : {
'created': { 'type': 'date', 'index': 'not_analyzed'},
'time': { 'type': 'string', 'index': 'not_analyzed'},
'day': { 'type': 'string', 'index': 'not_analyzed'},
'week': { 'type': 'string', 'index': 'not_analyzed'},
'month': { 'type': 'string', 'index': 'not_analyzed'},
'year': { 'type': 'string', 'index': 'not_analyzed'},
'app': { 'type': 'string', 'index': 'not_analyzed'},
'host': { 'type': 'string', 'index': 'not_analyzed'},
'eventType': { 'type': 'string', 'index': 'not_analyzed'},
'account': { 'type': 'string', 'index': 'not_analyzed'},
'uniqueName': { 'type': 'string', 'index': 'not_analyzed'},
'cost': { 'type': 'long', 'index': 'not_analyzed'},
'status': { 'type': 'string', 'index': 'not_analyzed'},
'messageSmart': { 'type': 'string', 'analyzer': 'ik_smart', 'search_analyzer': 'ik_smart', 'include_in_all': 'true', 'boost': 8},
'messageMax': { 'type': 'string', 'analyzer': 'ik_max_word', 'search_analyzer': 'ik_max_word', 'include_in_all': 'true', 'boost': 8}
}
}
}
}
es.create_index(index = index, settings = mapping)


if __name__ == '__main__':
main(sys.argv)
6 changes: 6 additions & 0 deletions skyeye-collector/skyeye-collector-metrics/src/main/resources/shell/install.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
#!/bin/bash

echo "--> install python-pip module"
sudo apt-get install python-pip
echo "--> install elasticsearch module"
sudo pip install pyelasticsearch
12 changes: 12 additions & 0 deletions skyeye-collector/skyeye-collector-metrics/src/main/resources/shell/start.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/bash

# 索引
# urls http://192.168.88.70:9200,http://192.168.88.71:9200,http://192.168.88.72:9200
# call: bash start.sh event-log http://192.168.88.70:9200,http://192.168.88.71:9200,http://192.168.88.72:9200

index=$1
urls=$2

echo '创建索引开始'
python create-index.py ${index} ${urls}
echo '创建索引结束'
3 changes: 3 additions & 0 deletions skyeye-collector/skyeye-collector-trace/src/main/resources/shell/hbase
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
create 'trace', {NAME => 'span', VERSIONS => 1, COMPRESSION => 'SNAPPY'}, {NUMREGIONS => 3, SPLITALGO => 'HexStringSplit'}
create 'time_consume', {NAME => 'trace', VERSIONS => 1, COMPRESSION => 'SNAPPY'}, {NUMREGIONS => 3, SPLITALGO => 'HexStringSplit'}
create 'annotation', {NAME => 'trace', VERSIONS => 1, COMPRESSION => 'SNAPPY'}, {NUMREGIONS => 3, SPLITALGO => 'HexStringSplit'}

0 comments on commit aeb304d

Please sign in to comment.