Truncate long sql queries to prevent oversized events to APM server (e…

…lastic#842)

* Truncate long sql queries to prevent oversized events to APM server

* Truncate sql to 10000 characters instead of keyword length

* Add a test

* Add to changelog

* Add encoding for sqlite_tests.py

* Use existing `shorten()` instead of `truncate()`

CHANGELOG.asciidoc

@@ -33,6 +33,10 @@ endif::[]
 
 * Added error handling around frame processing in Django {pull}837[#837]
 
+[float]
+===== Bug fixes
+
+* Limit SQL queries in context data to 10000 characters {pull}842[#842]
 
 [[release-notes-5.x]]
 === Python Agent version 5.x

elasticapm/instrumentation/packages/dbapi2.py

@@ -38,7 +38,7 @@
 from elasticapm.instrumentation.packages.base import AbstractInstrumentedModule
 from elasticapm.traces import capture_span
 from elasticapm.utils import compat, wrapt
-from elasticapm.utils.encoding import force_text
+from elasticapm.utils.encoding import force_text, shorten
 
 
 class Literal(object):
@@ -225,6 +225,11 @@ def _trace_sql(self, method, sql, params, action=QUERY_ACTION):
             signature = sql_string + "()"
         else:
             signature = self.extract_signature(sql_string)
+
+        # Truncate sql_string to 10000 characters to prevent large queries from
+        # causing an error to APM server.
+        sql_string = shorten(sql_string, string_length=10000)
+
         with capture_span(
             signature,
             span_type="db",

elasticapm/instrumentation/packages/sqlite.py

@@ -35,6 +35,7 @@
     extract_signature,
 )
 from elasticapm.traces import capture_span
+from elasticapm.utils.encoding import shorten
 
 
 class SQLiteCursorProxy(CursorProxy):
@@ -52,12 +53,13 @@ class SQLiteConnectionProxy(ConnectionProxy):
 
     def _trace_sql(self, method, sql, params):
         signature = extract_signature(sql)
+        sql_string = shorten(sql, string_length=10000)
         with capture_span(
             signature,
             span_type="db",
             span_subtype="sqlite",
             span_action="query",
-            extra={"db": {"type": "sql", "statement": sql}},
+            extra={"db": {"type": "sql", "statement": sql_string}},
         ):
             if params is None:
                 return method(sql)

elasticapm/utils/encoding.py

@@ -218,6 +218,10 @@ def shorten(var, list_length=50, string_length=200, dict_length=50):
 
 
 def keyword_field(string):
+    """
+    If the given string is longer than KEYWORD_MAX_LENGTH, truncate it to
+    KEYWORD_MAX_LENGTH-1, adding the "…" character at the end.
+    """
     if not isinstance(string, compat.string_types) or len(string) <= KEYWORD_MAX_LENGTH:
         return string
     return string[: KEYWORD_MAX_LENGTH - 1] + u"…"

tests/instrumentation/sqlite_tests.py

@@ -125,3 +125,20 @@ def test_nonstandard_connection_execute(instrument, elasticapm_client):
     assert spans[3]["action"] == "query"
 
     assert len(spans) == 4
+
+
+def test_truncate_long_sql(instrument, elasticapm_client):
+    conn = sqlite3.connect(":memory:")
+
+    conn.execute("CREATE TABLE testdb (id integer, username text)")
+    conn.execute('INSERT INTO testdb VALUES (1, "Ron")')
+    elasticapm_client.begin_transaction("transaction.test")
+    conn.execute("""SELECT id, username FROM testdb WHERE username = "%s";""" % ("x" * 10010))
+    elasticapm_client.end_transaction("MyView")
+    conn.execute("DROP TABLE testdb")
+
+    transactions = elasticapm_client.events[TRANSACTION]
+    spans = elasticapm_client.spans_for_transaction(transactions[0])
+
+    assert len(spans[0]["context"]["db"]["statement"]) == 10000
+    assert spans[0]["context"]["db"]["statement"].endswith("...")