Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Surface HTML errors #121

Merged
merged 6 commits into from
Feb 21, 2018
Merged

Surface HTML errors #121

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
eef3ed1
Extract error from HTML
betodealmeida Feb 14, 2018
49eaaca
Fixes
betodealmeida Feb 14, 2018
676b661
Fix JSONDecodeError
betodealmeida Feb 14, 2018
39088ad
Sort keys so that unit test is deterministic
betodealmeida Feb 14, 2018
c4af287
Remove trailing whitespace in encoded JSON
betodealmeida Feb 14, 2018
d4f7747
Use textwrap and remove JSONDecodeError
betodealmeida Feb 20, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 22 additions & 6 deletions pydruid/client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,23 @@
from __future__ import absolute_import

import json
import sys
import re

from six.moves import urllib

from pydruid.query import QueryBuilder
from base64 import b64encode

try:
# available only in Python >= 3.5
from json.decoder import JSONDecodeError
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks like JSONDecodeError is derived from ValueError, so catching ValueError should just work and remove the need for this extra complexity.

except ImportError:
JSONDecodeError = ValueError


# extract error from the <PRE> tag inside the HTML response
HTML_ERROR = re.compile('<pre>\s*(.*?)\s*</pre>', re.IGNORECASE)


class BaseDruidClient(object):
def __init__(self, url, endpoint):
Expand Down Expand Up @@ -484,18 +494,24 @@ def _post(self, query):
res = urllib.request.urlopen(req)
data = res.read().decode("utf-8")
res.close()
except urllib.error.HTTPError:
_, e, _ = sys.exc_info()
err = None
except urllib.error.HTTPError as e:
err = e.reason
if e.code == 500:
# has Druid returned an error?
try:
err = json.loads(e.read().decode("utf-8"))
err = json.loads(err)
except JSONDecodeError:
if HTML_ERROR.search(err):
err = HTML_ERROR.search(err).group(1)
except (ValueError, AttributeError, KeyError):
pass

raise IOError('{0} \n Druid Error: {1} \n Query is: {2}'.format(
e, err, json.dumps(query.query_dict, indent=4)))
e, err, json.dumps(
query.query_dict,
indent=4,
sort_keys=True,
separators=(',', ': '))))
else:
query.parse(data)
return query
Expand Down
70 changes: 70 additions & 0 deletions tests/test_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,76 @@ def test_druid_returns_error(self, mock_urlopen):
threshold=1,
context={"timeout": 1000})

@patch('pydruid.client.urllib.request.urlopen')
def test_druid_returns_html_error(self, mock_urlopen):
# given
message = """<html>
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

NIT: there's this trick indent long strings in code but not in the variable itself:

    s = textwrap.dedent"""\
    this
        wont
    be indented
    """

<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /druid/v2/. Reason:
<pre> javax.servlet.ServletException: java.lang.OutOfMemoryError: GC overhead limit exceeded</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.19.v20170502</a><hr/>
</body>
</html>"""
ex = urllib.error.HTTPError(None, 500, message, None, None)
mock_urlopen.side_effect = ex
client = create_client()

# when / then
with pytest.raises(IOError) as e:
client.topn(
datasource="testdatasource",
granularity="all",
intervals="2015-12-29/pt1h",
aggregations={"count": doublesum("count")},
dimension="user_name",
metric="count",
filter=Dimension("user_lang") == "en",
threshold=1,
context={"timeout": 1000})

assert str(e.value) == """HTTP Error 500: <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /druid/v2/. Reason:
<pre> javax.servlet.ServletException: java.lang.OutOfMemoryError: GC overhead limit exceeded</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.19.v20170502</a><hr/>
</body>
</html>
Druid Error: javax.servlet.ServletException: java.lang.OutOfMemoryError: GC overhead limit exceeded
Query is: {
"aggregations": [
{
"fieldName": "count",
"name": "count",
"type": "doubleSum"
}
],
"context": {
"timeout": 1000
},
"dataSource": "testdatasource",
"dimension": "user_name",
"filter": {
"dimension": "user_lang",
"type": "selector",
"value": "en"
},
"granularity": "all",
"intervals": "2015-12-29/pt1h",
"metric": "count",
"queryType": "topN",
"threshold": 1
}"""

@patch('pydruid.client.urllib.request.urlopen')
def test_druid_returns_results(self, mock_urlopen):
# given
Expand Down