Skip to content

Commit

Permalink
Fix tests
Browse files Browse the repository at this point in the history 
stale_data_in_es now returns two rows one for cases and one for
case_search
  • Loading branch information
@MartinRiese
MartinRiese committed Jun 24, 2024
1 parent c0da84b commit d809a86
Show file tree
Hide file tree
Showing 2 changed files with 28 additions and 23 deletions.
8 changes: 4 additions & 4 deletions corehq/apps/hqadmin/management/commands/stale_data_in_es.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,9 +174,9 @@ def run(cls, run_config):
all_domains_with_dedicated_search_index = settings.CASE_SEARCH_SUB_INDICES.keys()
else:
all_domains = [run_config.domain]
if domain_needs_search_index(run_config.domain.name):
all_domains_with_search_index = [run_config.domain.name]
if run_config.domain in settings.Case_SEARCH_SUB_INDICES:
if domain_needs_search_index(run_config.domain):
all_domains_with_search_index = [run_config.domain]
if run_config.domain in settings.CASE_SEARCH_SUB_INDICES:
all_domains_with_dedicated_search_index = [run_config.domain]

case_query = CaseES(for_export=True)
Expand Down Expand Up @@ -295,7 +295,7 @@ def _yield_missing_in_es(chunk):
form_id, (None, None, None))
if (es_modified_on, es_doc_type, es_domain) != (modified_on, doc_type, domain):
yield DataRow(doc_id=form_id, doc_type=doc_type, doc_subtype=xmlns, domain=domain,
es_date=es_modified_on, primary_date=modified_on)
index=FormES().index, es_date=es_modified_on, primary_date=modified_on)

@staticmethod
def _get_es_modified_dates_for_forms(form_ids):
Expand Down
43 changes: 24 additions & 19 deletions corehq/apps/hqadmin/tests/test_stale_data_in_es.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ class TestStaleDataInESSQL(TestCase):
project_name = 'sql-project'
case_type = 'patient'
form_xmlns = None
maxDiff = None

def test_no_output(self):
self._assert_in_sync(self._stale_data_in_es('form'))
Expand Down Expand Up @@ -62,15 +63,15 @@ def test_case_resume(self):
def _make_fake_es_check(num_to_process):
seen = []

def _fake_es_check(chunk):
def _fake_es_check(chunk, query):
chunk = list(chunk)
if len(seen) == num_to_process:
raise ExitEarlyException(seen)
seen.extend(chunk)
for case_id, case_type, modified_on, domain in chunk:
yield DataRow(
doc_id=case_id, doc_type='CommCareCase', doc_subtype=case_type, domain=domain,
es_date=None, primary_date=modified_on
index=query.index, es_date=None, primary_date=modified_on
)

return _fake_es_check
Expand All @@ -87,15 +88,15 @@ def call(num_to_process=None, expect_exception=None):
form, cases = self._submit_form(self.project.name, new_cases=4)

# process first 2 then raise exception
self._assert_not_in_sync(call(2, expect_exception=ExitEarlyException), rows=[
(case.case_id, 'CommCareCase', case.type, case.domain, None, case.server_modified_on)
for case in cases[:2]
self._assert_not_in_sync(call(4, expect_exception=ExitEarlyException), rows=[
(case.case_id, 'CommCareCase', case.type, case.domain, index, None, case.server_modified_on)
for case in cases[:2] for index in ['cases', 'case_search']
])

# process rest - should start at 3rd doc
self._assert_not_in_sync(call(4), rows=[
(case.case_id, 'CommCareCase', case.type, case.domain, None, case.server_modified_on)
for case in cases[2:]
(case.case_id, 'CommCareCase', case.type, case.domain, index, None, case.server_modified_on)
for case in cases[2:] for index in ['cases', 'case_search']
])

def test_form_resume(self):
Expand All @@ -112,7 +113,7 @@ def _fake_es_check(chunk):
for form_id, doc_type, xmlns, modified_on, domain in chunk:
yield DataRow(
doc_id=form_id, doc_type=doc_type, doc_subtype=xmlns, domain=domain,
es_date=None, primary_date=modified_on
index='forms', es_date=None, primary_date=modified_on
)

return _fake_es_check
Expand All @@ -132,13 +133,13 @@ def call(num_to_process=None, expect_exception=None):

# process first 2 then raise exception
self._assert_not_in_sync(call(2, expect_exception=ExitEarlyException), rows=[
(form.form_id, 'XFormInstance', form.xmlns, form.domain, None, form.received_on)
(form.form_id, 'XFormInstance', form.xmlns, form.domain, 'forms', None, form.received_on)
for form in forms[:2]
])

# process rest - should start at 3rd doc
self._assert_not_in_sync(call(4), rows=[
(form.form_id, 'XFormInstance', form.xmlns, form.domain, None, form.received_on)
(form.form_id, 'XFormInstance', form.xmlns, form.domain, 'forms', None, form.received_on)
for form in forms[2:]
])

Expand All @@ -148,23 +149,23 @@ def call():

form, cases = self._submit_form(self.project.name)
self._assert_not_in_sync(call(), rows=[
(form.form_id, 'XFormInstance', form.xmlns, form.domain, None, form.received_on)
(form.form_id, 'XFormInstance', form.xmlns, form.domain, 'forms', None, form.received_on)
])

self._send_forms_to_es([form])
self._assert_in_sync(call())

form.archive(trigger_signals=False)
self._assert_not_in_sync(call(), rows=[
(form.form_id, 'XFormArchived', form.xmlns, form.domain, form.received_on, form.received_on)
(form.form_id, 'XFormArchived', form.xmlns, form.domain, 'forms', form.received_on, form.received_on)
])

self._send_forms_to_es([form])
self._assert_in_sync(call())

form.unarchive(trigger_signals=False)
self._assert_not_in_sync(call(), rows=[
(form.form_id, 'XFormInstance', form.xmlns, form.domain, form.received_on, form.received_on)
(form.form_id, 'XFormInstance', form.xmlns, form.domain, 'forms', form.received_on, form.received_on)
])

self._send_forms_to_es([form])
Expand All @@ -175,7 +176,8 @@ def call():
return self._stale_data_in_es('case', **cmd_kwargs)
form, (case,) = self._submit_form(self.project.name, new_cases=1)
self._assert_not_in_sync(call(), rows=[
(case.case_id, 'CommCareCase', case.type, case.domain, "None,None,None", case.server_modified_on)
(case.case_id, 'CommCareCase', case.type, case.domain, 'cases', None, case.server_modified_on),
(case.case_id, 'CommCareCase', case.type, case.domain, 'case_search', None, case.server_modified_on)
])

self._send_cases_to_es([case])
Expand All @@ -184,8 +186,10 @@ def call():
old_date = case.server_modified_on
form, (case,) = self._submit_form(self.project.name, update_cases=[case])
self._assert_not_in_sync(call(), rows=[
(case.case_id, 'CommCareCase', case.type, case.domain,
f"{old_date},{old_date},None", case.server_modified_on)
(case.case_id, 'CommCareCase', case.type, case.domain, 'cases', old_date,
case.server_modified_on),
(case.case_id, 'CommCareCase', case.type, case.domain, 'case_search', old_date,
case.server_modified_on)
])

self._send_cases_to_es([case])
Expand All @@ -202,7 +206,8 @@ def call():
case.server_modified_on = pg_modified_on

self._assert_not_in_sync(call(), rows=[
(case.case_id, 'CommCareCase', case.type, case.domain, "None,None,None", case.server_modified_on)
(case.case_id, 'CommCareCase', case.type, case.domain, "cases", None, case.server_modified_on),
(case.case_id, 'CommCareCase', case.type, case.domain, "case_search", None, case.server_modified_on)
])

def _stale_data_in_es(self, *args, **kwargs):
Expand Down Expand Up @@ -272,14 +277,14 @@ def _delete_cases_from_es(cls, case_ids):
def _assert_in_sync(self, output):
self.assertEqual(
output,
'Doc ID\tDoc Type\tDoc Subtype\tDomain\tES Date\tCorrect Date\n'
'Doc ID\tDoc Type\tDoc Subtype\tDomain\tIndex\tES Date\tCorrect Date\n'
)

def _assert_not_in_sync(self, output, rows):
content = "".join('{}\n'.format('\t'.join(map(str, row))) for row in rows)
self.assertEqual(
output,
f'Doc ID\tDoc Type\tDoc Subtype\tDomain\tES Date\tCorrect Date\n{content}'
f'Doc ID\tDoc Type\tDoc Subtype\tDomain\tIndex\tES Date\tCorrect Date\n{content}'
)

@classmethod
Expand Down

0 comments on commit d809a86

Please sign in to comment.