Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support insecure TLS and only CA cert for Elasticsearch #1918

Merged
merged 4 commits into from
Nov 14, 2019
Merged

Support insecure TLS and only CA cert for Elasticsearch #1918

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
64226c5
Support insecure TLS and only CA cert in Elasticsearch scripts
pavolloffay Nov 12, 2019
f129762
nosec
pavolloffay Nov 13, 2019
2106a86
fmt
pavolloffay Nov 13, 2019
abf0df3
Test compilation of ES scripts and use CA in precedence of insecure
pavolloffay Nov 14, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,3 +29,5 @@ cmd/docs/*.1
cmd/docs/*.yaml
crossdock/crossdock-*
run-crossdock.log

__pycache__
5 changes: 5 additions & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,11 @@ storage-integration-test: go-gen
go clean -testcache
bash -c "set -e; set -o pipefail; $(GOTEST) $(STORAGE_PKGS) | $(COLORIZE)"

.PHONE: test-compile-es-scripts
test-compile-es-scripts:
docker run --rm -it -v ${PWD}:/tmp/jaeger python:3-alpine /usr/local/bin/python -m py_compile /tmp/jaeger/plugin/storage/es/esRollover.py
docker run --rm -it -v ${PWD}:/tmp/jaeger python:3-alpine /usr/local/bin/python -m py_compile /tmp/jaeger/plugin/storage/es/esCleaner.py

.PHONY: index-cleaner-integration-test
index-cleaner-integration-test: docker-images-elastic
# Expire tests results for storage integration tests since the environment might change
Expand Down
4 changes: 3 additions & 1 deletion pkg/es/config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -302,14 +302,16 @@ func (c *Configuration) getConfigOptions(logger *zap.Logger) ([]elastic.ClientOp
} else {
httpTransport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
// #nosec G402
TLSClientConfig: &tls.Config{InsecureSkipVerify: c.TLS.SkipHostVerify},
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

when no TLS is involved at all, would non-nill TLSClientConfig cause issues?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It should not godoc from http.Transport.TLSClientConfig

	// TLSClientConfig specifies the TLS configuration to use with
	// tls.Client.
	// If nil, the default configuration is used.
	// If non-nil, HTTP/2 support may not be enabled by default.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this godoc doesn't seem to clarify what happens to tls / no-tls. Perhaps it's just driven off the http scheme

}
if c.TLS.CaPath != "" {
ctls := &TLSConfig{CaPath: c.TLS.CaPath}
ca, err := ctls.loadCertificate()
if err != nil {
return nil, err
}
httpTransport.TLSClientConfig = &tls.Config{RootCAs: ca}
httpTransport.TLSClientConfig.RootCAs = ca
}

token := ""
Expand Down
30 changes: 18 additions & 12 deletions plugin/storage/es/esCleaner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,20 +23,10 @@ def main():
print('ES_TLS_CA ... Path to TLS CA file.')
print('ES_TLS_CERT ... Path to TLS certificate file.')
print('ES_TLS_KEY ... Path to TLS key file.')
print('ES_TLS_SKIP_HOST_VERIFY ... (insecure) Skip server\'s certificate chain and host name verification.')
sys.exit(1)

username = os.getenv("ES_USERNAME")
password = os.getenv("ES_PASSWORD")

if username is not None and password is not None:
client = elasticsearch.Elasticsearch(sys.argv[2:], http_auth=(username, password))
elif str2bool(os.getenv("ES_TLS", 'false')):
context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=os.getenv("ES_TLS_CA"))
context.load_cert_chain(certfile=os.getenv("ES_TLS_CERT"), keyfile=os.getenv("ES_TLS_KEY"))
client = elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context)
else:
client = elasticsearch.Elasticsearch(sys.argv[2:])

client = create_client(os.getenv("ES_USERNAME"), os.getenv("ES_PASSWORD"), str2bool(os.getenv("ES_TLS", 'false')), os.getenv("ES_TLS_CA"), os.getenv("ES_TLS_CERT"), os.getenv("ES_TLS_KEY"), str2bool(os.getenv("ES_TLS_SKIP_HOST_VERIFY", 'false')))
ilo = curator.IndexList(client)
empty_list(ilo, 'Elasticsearch has no indices')

Expand Down Expand Up @@ -102,5 +92,21 @@ def str2bool(v):
return v.lower() in ('true', '1')


def create_client(username, password, tls, ca, cert, key, skipHostVerify):
context = ssl.create_default_context()
if ca is not None:
context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=ca)
elif skipHostVerify:
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
if username is not None and password is not None:
return elasticsearch.Elasticsearch(sys.argv[2:], http_auth=(username, password), ssl_context=context)
elif tls:
context.load_cert_chain(certfile=cert, keyfile=key)
return elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context)
else:
return elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context)


if __name__ == "__main__":
main()
38 changes: 24 additions & 14 deletions plugin/storage/es/esRollover.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ def main():
print('ES_TLS_CA ... Path to TLS CA file.')
print('ES_TLS_CERT ... Path to TLS certificate file.')
print('ES_TLS_KEY ... Path to TLS key file.')
print('ES_TLS_SKIP_HOST_VERIFY ... (insecure) Skip server\'s certificate chain and host name verification.')
print('ES_VERSION ... The major Elasticsearch version. If not specified, the value will be auto-detected from Elasticsearch.')
print('init configuration:')
print('\tSHARDS ... the number of shards per index in Elasticsearch (default {}).'.format(SHARDS))
Expand All @@ -46,18 +47,7 @@ def main():
print('\tUNIT_COUNT ... count of UNITs (default {}).'.format(UNIT_COUNT))
sys.exit(1)

username = os.getenv("ES_USERNAME")
password = os.getenv("ES_PASSWORD")

if username is not None and password is not None:
client = elasticsearch.Elasticsearch(sys.argv[2:], http_auth=(username, password))
elif str2bool(os.getenv("ES_TLS", 'false')):
context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=os.getenv("ES_TLS_CA"))
context.load_cert_chain(certfile=os.getenv("ES_TLS_CERT"), keyfile=os.getenv("ES_TLS_KEY"))
client = elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context)
else:
client = elasticsearch.Elasticsearch(sys.argv[2:])

client = create_client(os.getenv("ES_USERNAME"), os.getenv("ES_PASSWORD"), str2bool(os.getenv("ES_TLS", 'false')), os.getenv("ES_TLS_CA"), os.getenv("ES_TLS_CERT"), os.getenv("ES_TLS_KEY"), str2bool(os.getenv("ES_TLS_SKIP_HOST_VERIFY", 'false')))
prefix = os.getenv('INDEX_PREFIX', '')
if prefix != '':
prefix += '-'
Expand Down Expand Up @@ -107,7 +97,7 @@ def perform_action(action, client, write_alias, read_alias, index_to_rollover, t
def create_index_template(template, template_name):
print('Creating index template {}'.format(template_name))
headers = {'Content-Type': 'application/json'}
s = get_request_session(os.getenv("ES_USERNAME"), os.getenv("ES_PASSWORD"), str2bool(os.getenv("ES_TLS", 'false')), os.getenv("ES_TLS_CA"), os.getenv("ES_TLS_CERT"), os.getenv("ES_TLS_KEY"))
s = get_request_session(os.getenv("ES_USERNAME"), os.getenv("ES_PASSWORD"), str2bool(os.getenv("ES_TLS", 'false')), os.getenv("ES_TLS_CA"), os.getenv("ES_TLS_CERT"), os.getenv("ES_TLS_KEY"), os.getenv("ES_TLS_SKIP_HOST_VERIFY", 'false'))
r = s.put(sys.argv[2] + '/_template/' + template_name, headers=headers, data=template)
print(r.text)
r.raise_for_status()
Expand Down Expand Up @@ -202,8 +192,12 @@ def empty_list(ilo, error_msg):
sys.exit(0)


def get_request_session(username, password, tls, ca, cert, key):
def get_request_session(username, password, tls, ca, cert, key, skipHostVerify):
session = requests.Session()
if ca is not None:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: should this be elif?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

good point I think they are mutually exclusive.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have changed it a bit. if CA is provided it will use that in the precedence of skip verify. I want to avoid accidentally using insecure connections.

session.verify = ca
elif skipHostVerify:
session.verify = False
if username is not None and password is not None:
session.auth = HTTPBasicAuth(username, password)
elif tls:
Expand All @@ -221,6 +215,22 @@ def get_version(client):
return esVersion


def create_client(username, password, tls, ca, cert, key, skipHostVerify):
context = ssl.create_default_context()
if ca is not None:
context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=ca)
elif skipHostVerify:
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
if username is not None and password is not None:
return elasticsearch.Elasticsearch(sys.argv[2:], http_auth=(username, password), ssl_context=context)
elif tls:
context.load_cert_chain(certfile=cert, keyfile=key)
return elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context)
else:
return elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context)


if __name__ == "__main__":
logging.getLogger().setLevel(logging.DEBUG)
main()
1 change: 1 addition & 0 deletions scripts/travis/es-integration-test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ echo "Executing token propatagion test"
make build-crossdock-ui-placeholder
GOOS=linux make build-query

make test-compile-es-scripts
SPAN_STORAGE_TYPE=elasticsearch ./cmd/query/query-linux --es.server-urls=http://127.0.0.1:9200 --es.tls=false --es.version=7 --query.bearer-token-propagation=true &
PID=$(echo $!)
make token-propagation-integration-test
Expand Down