diff --git a/.gitignore b/.gitignore index 28622ea83e..7def2b2901 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,5 @@ grr/server/grr_response_server/gui/static/fonts/ installers/ GRRlog.txt *.log +grr/server/grr_response_server/gui/static/third-party +grr/server/grr_response_server/gui/ui/.angular diff --git a/CHANGELOG b/CHANGELOG index 93415fdf77..cde49adfc6 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,14 @@ ## Upcoming release +* Renamed AdminUI.new_hunt_wizard.default_output_plugin to + AdminUI.new_hunt_wizard.default_output_plugins (note the "s" in the end). + The new option accepts a comma-separated list of names. +* Fully removed deprecated use_tsk flag. +* Removed deprecated plugin_args field from OutputPluginDescriptor. + +## 3.4.6.7 + * Introduced Server.grr_binaries_readonly configuration option (set to False by default). When set to True, binaries and python hacks can't be overriden or deleted. diff --git a/colab/grr_colab/client_test.py b/colab/grr_colab/client_test.py index 37dd13b685..7ba2efd091 100644 --- a/colab/grr_colab/client_test.py +++ b/colab/grr_colab/client_test.py @@ -31,8 +31,7 @@ class ClientTest(testing.ColabE2ETest): NONEXISTENT_CLIENT_ID = 'C.5555555555555555' def testWithId_ClientExists(self): - data_store.REL_DB.WriteClientMetadata( - client_id=ClientTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=ClientTest.FAKE_CLIENT_ID) client = grr_colab.Client.with_id(ClientTest.FAKE_CLIENT_ID) self.assertIsNotNone(client) @@ -47,8 +46,7 @@ def testWithId_NoSuchClient(self): def testWithHostname_SingleClient(self): hostname = 'user.loc.group.example.com' - data_store.REL_DB.WriteClientMetadata( - client_id=ClientTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=ClientTest.FAKE_CLIENT_ID) client = rdf_objects.ClientSnapshot(client_id=ClientTest.FAKE_CLIENT_ID) client.knowledge_base.fqdn = hostname @@ -64,10 +62,8 @@ def testWithHostname_MultipleClients(self): client_id1 = 'C.1111111111111111' client_id2 = 'C.1111111111111112' - data_store.REL_DB.WriteClientMetadata( - client_id=client_id1, fleetspeak_enabled=False) - data_store.REL_DB.WriteClientMetadata( - client_id=client_id2, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=client_id1) + data_store.REL_DB.WriteClientMetadata(client_id=client_id2) client = rdf_objects.ClientSnapshot(client_id=client_id1) client.knowledge_base.fqdn = hostname @@ -98,10 +94,8 @@ def testSearch_SingleKeyword(self): client_id1 = 'C.1111111111111111' client_id2 = 'C.1111111111111112' - data_store.REL_DB.WriteClientMetadata( - client_id=client_id1, fleetspeak_enabled=False) - data_store.REL_DB.WriteClientMetadata( - client_id=client_id2, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=client_id1) + data_store.REL_DB.WriteClientMetadata(client_id=client_id2) client = rdf_objects.ClientSnapshot(client_id=client_id1) client.startup_info.client_info.labels.append('foo') @@ -121,10 +115,8 @@ def testSearch_NoResults(self): client_id1 = 'C.1111111111111111' client_id2 = 'C.1111111111111112' - data_store.REL_DB.WriteClientMetadata( - client_id=client_id1, fleetspeak_enabled=False) - data_store.REL_DB.WriteClientMetadata( - client_id=client_id2, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=client_id1) + data_store.REL_DB.WriteClientMetadata(client_id=client_id2) client = rdf_objects.ClientSnapshot(client_id=client_id1) client.startup_info.client_info.labels.append('foo') @@ -143,10 +135,8 @@ def testSearch_MultipleResults(self): client_id1 = 'C.1111111111111111' client_id2 = 'C.1111111111111112' - data_store.REL_DB.WriteClientMetadata( - client_id=client_id1, fleetspeak_enabled=False) - data_store.REL_DB.WriteClientMetadata( - client_id=client_id2, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=client_id1) + data_store.REL_DB.WriteClientMetadata(client_id=client_id2) client = rdf_objects.ClientSnapshot(client_id=client_id1) client.startup_info.client_info.labels.append('foo') @@ -168,10 +158,8 @@ def testSearch_MultipleKeywords(self): client_id1 = 'C.1111111111111111' client_id2 = 'C.1111111111111112' - data_store.REL_DB.WriteClientMetadata( - client_id=client_id1, fleetspeak_enabled=False) - data_store.REL_DB.WriteClientMetadata( - client_id=client_id2, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=client_id1) + data_store.REL_DB.WriteClientMetadata(client_id=client_id2) client = rdf_objects.ClientSnapshot(client_id=client_id1) client.knowledge_base.fqdn = hostname @@ -190,16 +178,14 @@ def testSearch_MultipleKeywords(self): self.assertEqual(clients[0].id, client_id1) def testId(self): - data_store.REL_DB.WriteClientMetadata( - client_id=ClientTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=ClientTest.FAKE_CLIENT_ID) client = grr_colab.Client.with_id(ClientTest.FAKE_CLIENT_ID) self.assertEqual(ClientTest.FAKE_CLIENT_ID, client.id) def testHostname(self): hostname = 'hostname.loc.group.example.com' - data_store.REL_DB.WriteClientMetadata( - client_id=ClientTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=ClientTest.FAKE_CLIENT_ID) client = rdf_objects.ClientSnapshot(client_id=ClientTest.FAKE_CLIENT_ID) client.knowledge_base.fqdn = hostname @@ -210,8 +196,7 @@ def testHostname(self): @parser_test_lib.WithAllParsers def testHostname_AfterInterrogate(self): - data_store.REL_DB.WriteClientMetadata( - client_id=ClientTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=ClientTest.FAKE_CLIENT_ID) client = grr_colab.Client.with_id(ClientTest.FAKE_CLIENT_ID) client.interrogate() @@ -219,8 +204,7 @@ def testHostname_AfterInterrogate(self): def testIfaces(self): ifname = 'test_ifname' - data_store.REL_DB.WriteClientMetadata( - client_id=ClientTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=ClientTest.FAKE_CLIENT_ID) client = rdf_objects.ClientSnapshot(client_id=ClientTest.FAKE_CLIENT_ID) client.interfaces = [rdf_client_network.Interface(ifname=ifname)] @@ -232,8 +216,7 @@ def testIfaces(self): @parser_test_lib.WithAllParsers def testIfaces_AfterInterrogate(self): - data_store.REL_DB.WriteClientMetadata( - client_id=ClientTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=ClientTest.FAKE_CLIENT_ID) client = grr_colab.Client.with_id(ClientTest.FAKE_CLIENT_ID) client.interrogate() @@ -248,8 +231,7 @@ def testKnowledgebase(self): data_store.REL_DB.WriteGRRUser(users[0]) data_store.REL_DB.WriteGRRUser(users[1]) - data_store.REL_DB.WriteClientMetadata( - client_id=ClientTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=ClientTest.FAKE_CLIENT_ID) client = rdf_objects.ClientSnapshot(client_id=ClientTest.FAKE_CLIENT_ID) client.knowledge_base.fqdn = fqdn @@ -269,8 +251,7 @@ def testKnowledgebase(self): def testArch(self): arch = 'x42' - data_store.REL_DB.WriteClientMetadata( - client_id=ClientTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=ClientTest.FAKE_CLIENT_ID) client = rdf_objects.ClientSnapshot(client_id=ClientTest.FAKE_CLIENT_ID) client.arch = arch @@ -281,8 +262,7 @@ def testArch(self): @parser_test_lib.WithAllParsers def testArch_AfterInterrogate(self): - data_store.REL_DB.WriteClientMetadata( - client_id=ClientTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=ClientTest.FAKE_CLIENT_ID) client = grr_colab.Client.with_id(ClientTest.FAKE_CLIENT_ID) client.interrogate() @@ -290,8 +270,7 @@ def testArch_AfterInterrogate(self): def testKernel(self): kernel = '0.0.0' - data_store.REL_DB.WriteClientMetadata( - client_id=ClientTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=ClientTest.FAKE_CLIENT_ID) client = rdf_objects.ClientSnapshot(client_id=ClientTest.FAKE_CLIENT_ID) client.kernel = kernel @@ -302,8 +281,7 @@ def testKernel(self): @parser_test_lib.WithAllParsers def testKernel_AfterInterrogate(self): - data_store.REL_DB.WriteClientMetadata( - client_id=ClientTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=ClientTest.FAKE_CLIENT_ID) client = grr_colab.Client.with_id(ClientTest.FAKE_CLIENT_ID) client.interrogate() @@ -314,8 +292,7 @@ def testLabels(self): owner = 'test-user' data_store.REL_DB.WriteGRRUser('test-user') - data_store.REL_DB.WriteClientMetadata( - client_id=ClientTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=ClientTest.FAKE_CLIENT_ID) data_store.REL_DB.AddClientLabels(ClientTest.FAKE_CLIENT_ID, owner, labels) client = grr_colab.Client.with_id(ClientTest.FAKE_CLIENT_ID) @@ -325,7 +302,6 @@ def testFirstSeen(self): first_seen = rdfvalue.RDFDatetime.Now() data_store.REL_DB.WriteClientMetadata( client_id=ClientTest.FAKE_CLIENT_ID, - fleetspeak_enabled=False, first_seen=first_seen) client = grr_colab.Client.with_id(ClientTest.FAKE_CLIENT_ID) @@ -335,15 +311,13 @@ def testLastSeen(self): last_seen = rdfvalue.RDFDatetime.Now() data_store.REL_DB.WriteClientMetadata( client_id=ClientTest.FAKE_CLIENT_ID, - fleetspeak_enabled=False, last_ping=last_seen) client = grr_colab.Client.with_id(ClientTest.FAKE_CLIENT_ID) self.assertEqual(client.last_seen, last_seen.AsDatetime()) def testRequestApproval(self): - data_store.REL_DB.WriteClientMetadata( - client_id=ClientTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=ClientTest.FAKE_CLIENT_ID) data_store.REL_DB.WriteGRRUser('foo') client = grr_colab.Client.with_id(ClientTest.FAKE_CLIENT_ID) @@ -359,8 +333,7 @@ def testRequestApproval(self): self.assertEqual(approvals[0].reason, 'test') def testRequestApprovalAndWait(self): - data_store.REL_DB.WriteClientMetadata( - client_id=ClientTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=ClientTest.FAKE_CLIENT_ID) data_store.REL_DB.WriteGRRUser('foo') client = grr_colab.Client.with_id(ClientTest.FAKE_CLIENT_ID) @@ -397,8 +370,7 @@ def ProcessApproval(): @parser_test_lib.WithAllParsers def testInterrogate(self): - data_store.REL_DB.WriteClientMetadata( - client_id=ClientTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=ClientTest.FAKE_CLIENT_ID) client = grr_colab.Client.with_id(ClientTest.FAKE_CLIENT_ID) summary = client.interrogate() @@ -406,8 +378,7 @@ def testInterrogate(self): @testing.with_approval_checks def testInterrogate_WithoutApproval(self): - data_store.REL_DB.WriteClientMetadata( - client_id=ClientTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=ClientTest.FAKE_CLIENT_ID) client = grr_colab.Client.with_id(ClientTest.FAKE_CLIENT_ID) @@ -417,8 +388,7 @@ def testInterrogate_WithoutApproval(self): self.assertEqual(context.exception.client_id, ClientTest.FAKE_CLIENT_ID) def testPs(self): - data_store.REL_DB.WriteClientMetadata( - client_id=ClientTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=ClientTest.FAKE_CLIENT_ID) client = grr_colab.Client.with_id(ClientTest.FAKE_CLIENT_ID) ps = client.ps() @@ -426,8 +396,7 @@ def testPs(self): @testing.with_approval_checks def testPs_WithoutApproval(self): - data_store.REL_DB.WriteClientMetadata( - client_id=ClientTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=ClientTest.FAKE_CLIENT_ID) client = grr_colab.Client.with_id(ClientTest.FAKE_CLIENT_ID) @@ -437,8 +406,7 @@ def testPs_WithoutApproval(self): self.assertEqual(context.exception.client_id, ClientTest.FAKE_CLIENT_ID) def testOsquery(self): - data_store.REL_DB.WriteClientMetadata( - client_id=ClientTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=ClientTest.FAKE_CLIENT_ID) client = grr_colab.Client.with_id(ClientTest.FAKE_CLIENT_ID) @@ -459,8 +427,7 @@ def testOsquery(self): @testing.with_approval_checks def testOsquery_WithoutApproval(self): - data_store.REL_DB.WriteClientMetadata( - client_id=ClientTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=ClientTest.FAKE_CLIENT_ID) client = grr_colab.Client.with_id(ClientTest.FAKE_CLIENT_ID) @@ -471,8 +438,7 @@ def testOsquery_WithoutApproval(self): @parser_test_lib.WithAllParsers def testCollect(self): - data_store.REL_DB.WriteClientMetadata( - client_id=ClientTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=ClientTest.FAKE_CLIENT_ID) client = rdf_objects.ClientSnapshot(client_id=ClientTest.FAKE_CLIENT_ID) client.knowledge_base.os = 'test-os' @@ -498,8 +464,7 @@ def testCollect(self): @testing.with_approval_checks def testCollect_WithoutApproval(self): - data_store.REL_DB.WriteClientMetadata( - client_id=ClientTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=ClientTest.FAKE_CLIENT_ID) client = grr_colab.Client.with_id(ClientTest.FAKE_CLIENT_ID) @@ -510,8 +475,7 @@ def testCollect_WithoutApproval(self): def testYara(self): search_str = 'foobarbaz-test-with-unique-string-in-memory' - data_store.REL_DB.WriteClientMetadata( - client_id=ClientTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=ClientTest.FAKE_CLIENT_ID) client = grr_colab.Client.with_id(ClientTest.FAKE_CLIENT_ID) diff --git a/colab/grr_colab/fs_test.py b/colab/grr_colab/fs_test.py index 1b610c0c1c..6fb131f610 100644 --- a/colab/grr_colab/fs_test.py +++ b/colab/grr_colab/fs_test.py @@ -21,7 +21,8 @@ class FileSystemTest(testing.ColabE2ETest): def testLs_ContainsFiles(self): data_store.REL_DB.WriteClientMetadata( - client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + client_id=FileSystemTest.FAKE_CLIENT_ID + ) dir_nodes = [ # name, content @@ -51,7 +52,8 @@ def testLs_ContainsFiles(self): def testLs_EmptyDirectory(self): data_store.REL_DB.WriteClientMetadata( - client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + client_id=FileSystemTest.FAKE_CLIENT_ID + ) with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath: fs_obj = fs.FileSystem(self._get_fake_api_client(), jobs_pb2.PathSpec.OS) @@ -61,7 +63,8 @@ def testLs_EmptyDirectory(self): def testLs_Recursive(self): data_store.REL_DB.WriteClientMetadata( - client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + client_id=FileSystemTest.FAKE_CLIENT_ID + ) dir_nodes = [ 'file0', @@ -98,7 +101,8 @@ def testLs_Recursive(self): def testLs_MaxDepth(self): data_store.REL_DB.WriteClientMetadata( - client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + client_id=FileSystemTest.FAKE_CLIENT_ID + ) dir_components = ['dir1', 'dir2', 'dir3', 'dir4', 'dir5'] @@ -121,7 +125,8 @@ def testLs_MaxDepth(self): @testing.with_approval_checks def testLs_WithoutApproval(self): data_store.REL_DB.WriteClientMetadata( - client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + client_id=FileSystemTest.FAKE_CLIENT_ID + ) fs_obj = fs.FileSystem(self._get_fake_api_client(), jobs_pb2.PathSpec.OS) @@ -132,7 +137,8 @@ def testLs_WithoutApproval(self): def testGlob_SingleFile(self): data_store.REL_DB.WriteClientMetadata( - client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + client_id=FileSystemTest.FAKE_CLIENT_ID + ) with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath: os.mkdir(os.path.join(temp_dirpath, 'dir')) @@ -148,7 +154,8 @@ def testGlob_SingleFile(self): def testGlob_MultipleFiles(self): data_store.REL_DB.WriteClientMetadata( - client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + client_id=FileSystemTest.FAKE_CLIENT_ID + ) with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath: os.mkdir(os.path.join(temp_dirpath, 'dir')) @@ -171,7 +178,8 @@ def testGlob_MultipleFiles(self): def testGlob_NoFiles(self): data_store.REL_DB.WriteClientMetadata( - client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + client_id=FileSystemTest.FAKE_CLIENT_ID + ) with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath: fs_obj = fs.FileSystem(self._get_fake_api_client(), jobs_pb2.PathSpec.OS) @@ -182,7 +190,8 @@ def testGlob_NoFiles(self): @testing.with_approval_checks def testGlob_WithoutApproval(self): data_store.REL_DB.WriteClientMetadata( - client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + client_id=FileSystemTest.FAKE_CLIENT_ID + ) fs_obj = fs.FileSystem(self._get_fake_api_client(), jobs_pb2.PathSpec.OS) @@ -193,7 +202,8 @@ def testGlob_WithoutApproval(self): def testGrep_HasMatches(self): data_store.REL_DB.WriteClientMetadata( - client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + client_id=FileSystemTest.FAKE_CLIENT_ID + ) filename = 'foo' with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath: @@ -216,7 +226,8 @@ def testGrep_HasMatches(self): def testGrep_Regex(self): data_store.REL_DB.WriteClientMetadata( - client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + client_id=FileSystemTest.FAKE_CLIENT_ID + ) filename = 'foo' with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath: @@ -237,7 +248,8 @@ def testGrep_Regex(self): def testGrep_NoMatches(self): data_store.REL_DB.WriteClientMetadata( - client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + client_id=FileSystemTest.FAKE_CLIENT_ID + ) filename = 'foo' with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath: @@ -252,7 +264,8 @@ def testGrep_NoMatches(self): def testGrep_BinaryPattern(self): data_store.REL_DB.WriteClientMetadata( - client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + client_id=FileSystemTest.FAKE_CLIENT_ID + ) filename = 'foo' with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath: @@ -272,7 +285,8 @@ def testGrep_BinaryPattern(self): @testing.with_approval_checks def testGrep_WithoutApproval(self): data_store.REL_DB.WriteClientMetadata( - client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + client_id=FileSystemTest.FAKE_CLIENT_ID + ) fs_obj = fs.FileSystem(self._get_fake_api_client(), jobs_pb2.PathSpec.OS) @@ -283,7 +297,8 @@ def testGrep_WithoutApproval(self): def testFgrep_HasMatches(self): data_store.REL_DB.WriteClientMetadata( - client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + client_id=FileSystemTest.FAKE_CLIENT_ID + ) filename = 'foo' with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath: @@ -302,7 +317,8 @@ def testFgrep_HasMatches(self): def testFgrep_Regex(self): data_store.REL_DB.WriteClientMetadata( - client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + client_id=FileSystemTest.FAKE_CLIENT_ID + ) filename = 'foo' with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath: @@ -319,7 +335,8 @@ def testFgrep_Regex(self): def testFgrep_NoMatches(self): data_store.REL_DB.WriteClientMetadata( - client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + client_id=FileSystemTest.FAKE_CLIENT_ID + ) filename = 'foo' with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath: @@ -334,7 +351,8 @@ def testFgrep_NoMatches(self): def testFgrep_BinaryPattern(self): data_store.REL_DB.WriteClientMetadata( - client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + client_id=FileSystemTest.FAKE_CLIENT_ID + ) filename = 'foo' with temp.AutoTempDirPath(remove_non_empty=True) as temp_dirpath: @@ -352,7 +370,8 @@ def testFgrep_BinaryPattern(self): @testing.with_approval_checks def testFgrep_WithoutApproval(self): data_store.REL_DB.WriteClientMetadata( - client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + client_id=FileSystemTest.FAKE_CLIENT_ID + ) fs_obj = fs.FileSystem(self._get_fake_api_client(), jobs_pb2.PathSpec.OS) @@ -364,7 +383,8 @@ def testFgrep_WithoutApproval(self): @testing.with_approval_checks def testWget_WithoutApproval(self): data_store.REL_DB.WriteClientMetadata( - client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + client_id=FileSystemTest.FAKE_CLIENT_ID + ) fs_obj = fs.FileSystem(self._get_fake_api_client(), jobs_pb2.PathSpec.OS) @@ -376,7 +396,8 @@ def testWget_WithoutApproval(self): def testWget_NoAdminURLSpecified(self): data_store.REL_DB.WriteClientMetadata( - client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + client_id=FileSystemTest.FAKE_CLIENT_ID + ) fs_obj = fs.FileSystem(self._get_fake_api_client(), jobs_pb2.PathSpec.OS) with flagsaver.flagsaver(grr_admin_ui_url=''): @@ -387,7 +408,8 @@ def testWget_NoAdminURLSpecified(self): def testWget_FileDoesNotExist(self): data_store.REL_DB.WriteClientMetadata( - client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + client_id=FileSystemTest.FAKE_CLIENT_ID + ) fs_obj = fs.FileSystem(self._get_fake_api_client(), jobs_pb2.PathSpec.OS) with flagsaver.flagsaver(grr_admin_ui_url=self.endpoint): @@ -396,7 +418,8 @@ def testWget_FileDoesNotExist(self): def testWget_IsDirectory(self): data_store.REL_DB.WriteClientMetadata( - client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + client_id=FileSystemTest.FAKE_CLIENT_ID + ) fs_obj = fs.FileSystem(self._get_fake_api_client(), jobs_pb2.PathSpec.OS) with flagsaver.flagsaver(grr_admin_ui_url=self.endpoint): @@ -407,7 +430,8 @@ def testWget_IsDirectory(self): def testWget_LinkWorksWithOfflineClient(self): data_store.REL_DB.WriteClientMetadata( - client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + client_id=FileSystemTest.FAKE_CLIENT_ID + ) fs_obj = fs.FileSystem(self._get_fake_api_client(), jobs_pb2.PathSpec.OS) content = b'foo bar' @@ -422,7 +446,8 @@ def testWget_LinkWorksWithOfflineClient(self): def testOpen_ReadAll(self): data_store.REL_DB.WriteClientMetadata( - client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + client_id=FileSystemTest.FAKE_CLIENT_ID + ) filename = 'foo' content = b'foo bar' @@ -437,7 +462,8 @@ def testOpen_ReadAll(self): def testOpen_ReadMore(self): data_store.REL_DB.WriteClientMetadata( - client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + client_id=FileSystemTest.FAKE_CLIENT_ID + ) filename = 'foo' content = b'foo bar' @@ -452,7 +478,8 @@ def testOpen_ReadMore(self): def testOpen_ReadLess(self): data_store.REL_DB.WriteClientMetadata( - client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + client_id=FileSystemTest.FAKE_CLIENT_ID + ) filename = 'foo' content = b'foo bar' @@ -467,7 +494,8 @@ def testOpen_ReadLess(self): def testOpen_Buffering(self): data_store.REL_DB.WriteClientMetadata( - client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + client_id=FileSystemTest.FAKE_CLIENT_ID + ) filename = 'foo' size = 1024 * 1024 @@ -484,7 +512,8 @@ def testOpen_Buffering(self): def testOpen_ReadLargeFile(self): data_store.REL_DB.WriteClientMetadata( - client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + client_id=FileSystemTest.FAKE_CLIENT_ID + ) filename = 'foo' size = 1024 * 1024 @@ -499,7 +528,8 @@ def testOpen_ReadLargeFile(self): def testOpen_SeekWithinOneBuffer(self): data_store.REL_DB.WriteClientMetadata( - client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + client_id=FileSystemTest.FAKE_CLIENT_ID + ) content = b'foo bar' with temp.AutoTempFilePath() as temp_filepath: @@ -515,7 +545,8 @@ def testOpen_SeekWithinOneBuffer(self): def testOpen_SeekOutOfBuffer(self): data_store.REL_DB.WriteClientMetadata( - client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + client_id=FileSystemTest.FAKE_CLIENT_ID + ) size = 1024 * 512 with temp.AutoTempFilePath() as temp_filepath: @@ -532,7 +563,8 @@ def testOpen_SeekOutOfBuffer(self): @testing.with_approval_checks def testOpen_WithoutApproval(self): data_store.REL_DB.WriteClientMetadata( - client_id=FileSystemTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + client_id=FileSystemTest.FAKE_CLIENT_ID + ) fs_obj = fs.FileSystem(self._get_fake_api_client(), jobs_pb2.PathSpec.OS) diff --git a/colab/grr_colab/vfs_test.py b/colab/grr_colab/vfs_test.py index 7332565e2d..13d06431f2 100644 --- a/colab/grr_colab/vfs_test.py +++ b/colab/grr_colab/vfs_test.py @@ -24,8 +24,7 @@ class VfsTest(testing.ColabE2ETest): @testing.with_approval_checks def testOpen_WithoutApproval(self): - data_store.REL_DB.WriteClientMetadata( - client_id=VfsTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=VfsTest.FAKE_CLIENT_ID) vfs_obj = vfs.VFS(self._get_fake_api_client(), jobs_pb2.PathSpec.OS) @@ -35,8 +34,7 @@ def testOpen_WithoutApproval(self): self.assertEqual(context.exception.client_id, VfsTest.FAKE_CLIENT_ID) def testOpen_DoesNotExist(self): - data_store.REL_DB.WriteClientMetadata( - client_id=VfsTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=VfsTest.FAKE_CLIENT_ID) vfs_obj = vfs.VFS(self._get_fake_api_client(), jobs_pb2.PathSpec.OS) @@ -44,8 +42,7 @@ def testOpen_DoesNotExist(self): vfs_obj.open('/foo/bar') def testOpen_NotCollected(self): - data_store.REL_DB.WriteClientMetadata( - client_id=VfsTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=VfsTest.FAKE_CLIENT_ID) content = b'foo bar' api_client = self._get_fake_api_client() @@ -62,8 +59,7 @@ def testOpen_NotCollected(self): vfs_obj.open(temp_filepath) def testOpen_ReadAll(self): - data_store.REL_DB.WriteClientMetadata( - client_id=VfsTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=VfsTest.FAKE_CLIENT_ID) content = b'foo bar' api_client = self._get_fake_api_client() @@ -81,8 +77,7 @@ def testOpen_ReadAll(self): self.assertEqual(filedesc.read(), content) def testOpen_ReadMore(self): - data_store.REL_DB.WriteClientMetadata( - client_id=VfsTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=VfsTest.FAKE_CLIENT_ID) content = b'foo bar' api_client = self._get_fake_api_client() @@ -100,8 +95,7 @@ def testOpen_ReadMore(self): self.assertEqual(filedesc.read(10), content) def testOpen_ReadLess(self): - data_store.REL_DB.WriteClientMetadata( - client_id=VfsTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=VfsTest.FAKE_CLIENT_ID) content = b'foo bar' api_client = self._get_fake_api_client() @@ -119,8 +113,7 @@ def testOpen_ReadLess(self): self.assertEqual(filedesc.read(3), b'foo') def testOpen_Buffering(self): - data_store.REL_DB.WriteClientMetadata( - client_id=VfsTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=VfsTest.FAKE_CLIENT_ID) size = 1024 * 1024 api_client = self._get_fake_api_client() @@ -140,8 +133,7 @@ def testOpen_Buffering(self): self.assertGreater(filedesc.tell(), 0) def testOpen_ReadLargeFile(self): - data_store.REL_DB.WriteClientMetadata( - client_id=VfsTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=VfsTest.FAKE_CLIENT_ID) size = 1024 * 1024 api_client = self._get_fake_api_client() @@ -159,8 +151,7 @@ def testOpen_ReadLargeFile(self): self.assertEqual(len(filedesc.read()), size) def testOpen_SeekWithinOneBuffer(self): - data_store.REL_DB.WriteClientMetadata( - client_id=VfsTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=VfsTest.FAKE_CLIENT_ID) content = b'foo bar' api_client = self._get_fake_api_client() @@ -180,8 +171,7 @@ def testOpen_SeekWithinOneBuffer(self): self.assertEqual(filedesc.read(), b'bar') def testOpen_SeekOutOfBuffer(self): - data_store.REL_DB.WriteClientMetadata( - client_id=VfsTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=VfsTest.FAKE_CLIENT_ID) size = 1024 * 512 api_client = self._get_fake_api_client() @@ -202,8 +192,7 @@ def testOpen_SeekOutOfBuffer(self): @testing.with_approval_checks def testLs_WithoutApproval(self): - data_store.REL_DB.WriteClientMetadata( - client_id=VfsTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=VfsTest.FAKE_CLIENT_ID) vfs_obj = vfs.VFS(self._get_fake_api_client(), jobs_pb2.PathSpec.OS) with self.assertRaises(errors.ApprovalMissingError) as context: @@ -212,16 +201,14 @@ def testLs_WithoutApproval(self): self.assertEqual(context.exception.client_id, VfsTest.FAKE_CLIENT_ID) def testLs_DoesNotExist(self): - data_store.REL_DB.WriteClientMetadata( - client_id=VfsTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=VfsTest.FAKE_CLIENT_ID) vfs_obj = vfs.VFS(self._get_fake_api_client(), jobs_pb2.PathSpec.OS) with self.assertRaises(api_errors.ResourceNotFoundError): vfs_obj.ls('/foo/bar') def testLs_ContainsFiles(self): - data_store.REL_DB.WriteClientMetadata( - client_id=VfsTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=VfsTest.FAKE_CLIENT_ID) dir_nodes = [ # name, content @@ -254,8 +241,7 @@ def testLs_ContainsFiles(self): self.assertEqual(stat_entries[1].st_size, 7) def testLs_EmptyDirectory(self): - data_store.REL_DB.WriteClientMetadata( - client_id=VfsTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=VfsTest.FAKE_CLIENT_ID) api_client = self._get_fake_api_client() client = grr_colab.Client(api_client) @@ -268,8 +254,7 @@ def testLs_EmptyDirectory(self): self.assertEmpty(stat_entries) def testLs_NotDirectory(self): - data_store.REL_DB.WriteClientMetadata( - client_id=VfsTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=VfsTest.FAKE_CLIENT_ID) api_client = self._get_fake_api_client() client = grr_colab.Client(api_client) @@ -282,8 +267,7 @@ def testLs_NotDirectory(self): vfs_obj.ls(temp_file) def testLs_Recursive(self): - data_store.REL_DB.WriteClientMetadata( - client_id=VfsTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=VfsTest.FAKE_CLIENT_ID) dir_nodes = [ 'file0', @@ -317,8 +301,7 @@ def testLs_Recursive(self): self.assertEqual(paths[5], os.path.join(temp_dirpath, 'file0')) def testLs_MaxDepth(self): - data_store.REL_DB.WriteClientMetadata( - client_id=VfsTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=VfsTest.FAKE_CLIENT_ID) dir_components = ['dir1', 'dir2', 'dir3', 'dir4', 'dir5'] @@ -342,8 +325,7 @@ def testLs_MaxDepth(self): @testing.with_approval_checks def testRefresh_WithoutApproval(self): - data_store.REL_DB.WriteClientMetadata( - client_id=VfsTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=VfsTest.FAKE_CLIENT_ID) vfs_obj = vfs.VFS(self._get_fake_api_client(), jobs_pb2.PathSpec.OS) @@ -353,16 +335,14 @@ def testRefresh_WithoutApproval(self): self.assertEqual(context.exception.client_id, VfsTest.FAKE_CLIENT_ID) def testRefresh_DoesNotExist(self): - data_store.REL_DB.WriteClientMetadata( - client_id=VfsTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=VfsTest.FAKE_CLIENT_ID) vfs_obj = vfs.VFS(self._get_fake_api_client(), jobs_pb2.PathSpec.OS) with self.assertRaises(api_errors.ResourceNotFoundError): vfs_obj.refresh('/foo/bar') def testRefresh_Plain(self): - data_store.REL_DB.WriteClientMetadata( - client_id=VfsTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=VfsTest.FAKE_CLIENT_ID) api_client = self._get_fake_api_client() client = grr_colab.Client(api_client) @@ -389,8 +369,7 @@ def testRefresh_Plain(self): self.assertEqual(paths[1], os.path.join(temp_dirpath, 'dir2')) def testRefresh_Recursive(self): - data_store.REL_DB.WriteClientMetadata( - client_id=VfsTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=VfsTest.FAKE_CLIENT_ID) dir_components = ['dir1', 'dir2', 'dir3', 'dir4', 'dir5'] @@ -416,8 +395,7 @@ def testRefresh_Recursive(self): @testing.with_approval_checks def testWget_WithoutApproval(self): - data_store.REL_DB.WriteClientMetadata( - client_id=VfsTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=VfsTest.FAKE_CLIENT_ID) vfs_obj = vfs.VFS(self._get_fake_api_client(), jobs_pb2.PathSpec.OS) @@ -428,8 +406,7 @@ def testWget_WithoutApproval(self): self.assertEqual(context.exception.client_id, VfsTest.FAKE_CLIENT_ID) def testWget_NoAdminURLSpecified(self): - data_store.REL_DB.WriteClientMetadata( - client_id=VfsTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=VfsTest.FAKE_CLIENT_ID) api_client = self._get_fake_api_client() client = grr_colab.Client(api_client) @@ -447,8 +424,7 @@ def testWget_NoAdminURLSpecified(self): vfs_obj.wget(temp_file) def testWget_FileDoesNotExist(self): - data_store.REL_DB.WriteClientMetadata( - client_id=VfsTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=VfsTest.FAKE_CLIENT_ID) vfs_obj = vfs.VFS(self._get_fake_api_client(), jobs_pb2.PathSpec.OS) @@ -457,8 +433,7 @@ def testWget_FileDoesNotExist(self): vfs_obj.wget('/non/existing/file') def testWget_IsDirectory(self): - data_store.REL_DB.WriteClientMetadata( - client_id=VfsTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=VfsTest.FAKE_CLIENT_ID) api_client = self._get_fake_api_client() client = grr_colab.Client(api_client) @@ -472,8 +447,7 @@ def testWget_IsDirectory(self): vfs_obj.wget(temp_dir) def testWget_LinkWorksWithOfflineClient(self): - data_store.REL_DB.WriteClientMetadata( - client_id=VfsTest.FAKE_CLIENT_ID, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id=VfsTest.FAKE_CLIENT_ID) api_client = self._get_fake_api_client() client = grr_colab.Client(api_client) diff --git a/devenv/README.md b/devenv/README.md new file mode 100644 index 0000000000..dd10ef66b6 --- /dev/null +++ b/devenv/README.md @@ -0,0 +1,154 @@ +# GRR Development Environment + +## TL;DR + +```bash +$ git clone https://github.com/google/grr +$ cd grr +$ devenv/devenv.sh check_deps +$ devenv/devenv.sh start +``` + +## Overview + +The dev environment relies on [podman](https://podman.io) to set up and run a +local GRR deployment. Each GRR component is run in its own container, and all +containers are bundled together in a pod (so that they share a network stack, +isolated from the host). All containers are run rootless, so Podman needs to be +available on the development machine and subuids need to be set up for the +development user. E.g.: + +```bash +sudo usermod --add-subuids 500000-565535 --add-subgids 500000-565535 $USER +``` + +Services, such as the GRR Admin UI and both the GRR and Fleetspeak MySQL DBs, +are exposed to the host via container port forwarding, so that they can be +accessed directly during development. + +The dev environment is set up and controlled via subcommands of the `devenv.sh` +tool (see `devenv/devenv.sh --help` for a description of the full functionality +available). + +## Environment Breakdown + +The dev environment will run a Fleetspeak-enabled GRR deployment, with a single +GRR client. Each component is run in its own container. + +### Containers + +- MySQL (`resdefs.MYSQL_CTR`): managing both the `grr` and `fleetspeak` DBs +- Fleetspeak Admin (`resdefs.FLEETSPEAK_ADMIN_CTR`) +- Fleetspeak Frontend (`resdefs.FLEETSPEAK_FRONTEND_CTR`) +- GRR AdminUI (`resdefs.GRR_ADMIN_UI_CTR`) +- GRR Worker (`resdefs.GRR_WORKER_CTR`) +- GRR Fleetspeak Frontend (`resdefs.GRR_FRONTEND_CTR`) +- GRR Client (`resdefs.GRR_CLIENT_CTR`): running the fleetspeak client, which + spawns the GRR client process + +### Persistent Data + +Data that needs to persist between devenv invocations is mounted inside the +containers as persistent volumes: +- `resdefs.MYSQL_DATA_VOL`: mounted inside the MySQL container (at + `/var/lib/mysql`), holding both the `grr` and `fleetspeak` DBs; +- `resdefs.GRR_PERSIST_VOL`: mounted insde all GRR containers (at + `/grr/persist`), holding the GRR Python virtual environment (i.e. build data + and deps), and the Fleetspeak client state file. + +### GRR Source Code + +The GRR source code is bind-mounted inside the GRR containers so that code code +changes are immediately available. Note however that, upon code changes, a +container restart is needed so that Python can pick up the changes (i.e +`devenv.sh restart {container}`). + +### GRR Configuration + +The config used by all GRR containers resides under `devenv/config`. As the +entire GRR source tree is bind-mounted into all the GRR containers, this +directory is also available, at runtime, to all GRR components. + +## Development Flow Example + +1. check that the dev environment can run on the host system: `bash + devenv/devenv.sh check_deps` +2. start the dev environment: `bash devenv/devenv.sh start` +3. check that everything is up and running: `bash devenv/devenv.sh status` +4. find the generated GRR client ID: `bash curl -su admin:admin + http://localhost:4280/api/clients \ | sed 1d \ | jq -r + ".items[].value.client_id.value"` Note: the above assumes the default values + in `devenv/config.py` (such as Admin UI port number and admin user details) + have not been changed. It also assumes `curl`, `sed`, and `jq` are available + on the host system. +5. open a browser and go to the Admin UI client info page: + `http://localhost:4280/#/clients/{CLIENT_ID}/host-info` +6. edit the GRR worker python code; +7. restart the `grr-worker` container so that code changes are picked up: + `devenv/devenv.sh restart grr-worker` + +### Debugging + +(still assuming GRR worker) + +- add a `breakpoint()` call at the targeted location inside the GRR worker + python code; +- restart the GRR worker container, attaching to its TTY, and iteract with + `pdb`: `bash devenv/devenv.sh restart -a grr-worker` Notice the `-a` option + (attach). + +### Meta: Developing the Development Environment + +## Resources + +The dev environment is built around the concept of resources - everything is a +resource that can be either up (active) or down (inactive). A resource may need +other resources to be active (up) before it itself can be brought up. In other +words, resources take dependencies on other resources. For instance, the MySQL +container will need the MySQL container image to be built before it can start. + +Each resource is implemented by extending the abstract base class +`reslib.Resource`. The specific resource then needs to implement: + +- `Resource.is_up()`: check whether the resource is up or down; +- `Resource.create()`: bring up / create the resource; +- `Resource.destroy()`: take down / destroy the resource. + +If the resource needs to specify dependencies it can do so by passing a list of +those to the base `Resource` constructor (via the `deps` argument). + +Additional functionality is provided by the base `Resource` class, such as: + +- bringing up a resource by ensuring that all its dependencies are up before + creating the resource itself (implemented by `Resource.ensure()`); +- destroying the resource if necessary (implemented by `Resource.clean()`); +- destroying a resource and its dependencies (`Resource.deep_clean()`). + +Resources are implemented by two modules: + +- `reslib.py`: contains resource functionality (e.g. handing Podman data + volumes, container images, and containers); +- `resdefs.py`: contains the actual resource definitions (i.e. the volumes, + images, containers, etc, that make up the dev environment). + +The dev environment itself is implemented via a `Resource` that depends on all +GRR containers being up, as well as an admin user being set up for the GRR Admin +UI. This way, bringing up the environment is a just a mater of calling this +resource's `ensure()` function. + +## Command Line Interface + +All dev environment functionality is implemented via subcommands of `devenv.sh`. +`devenv.sh` itself is a wrapper script that runs the `devenv` Python package +in-place (i.e. without the need to py-install it). + +Command line parsing is done via the standard Python `argparse` module, in +`cli.py`. Each subcommand is implemented via a function defined in `commands.py`, +and decorated with `cli.subcommand`. This decorator ensures that the function is +register as a subcommand and takes care of parsing its command line. The +subcommand function will receive its parsed arguments in the form of a +`argparse.Namespace` object. + +The `cli.subcommand` decorator receives the parsing instructions as arguments. +These are passed through almost verbatim to `argparse` functions (see +`subcommand` code doc for the exact format). diff --git a/devenv/config/fleetspeak-client/client.config b/devenv/config/fleetspeak-client/client.config new file mode 100644 index 0000000000..083a3041d8 --- /dev/null +++ b/devenv/config/fleetspeak-client/client.config @@ -0,0 +1,8 @@ +trusted_certs: "-----BEGIN CERTIFICATE-----\nMIIBrjCCAVSgAwIBAgIUS8640c9otFFdzIXYRJ5iQ6qXypswCgYIKoZIzj0EAwIw\nFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTIzMDQyNzA4MjkxOFoXDTMzMDQyNDA4\nMjkxOFowFDESMBAGA1UEAwwJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0D\nAQcDQgAE0i/hPNwkQd9AdtCz7Idh38K1AefNjHrzIkgn+eQM/Z6+3AULd+ngYsPR\nfdkPjSRtxQ5TwL+oDddAJORi6r4Et6OBgzCBgDAtBgNVHREEJjAkghdzdmMtZmxl\nZXRzcGVhay1mcm9udGVuZIIJbG9jYWxob3N0MB0GA1UdDgQWBBT8hgEclgZe9TLr\nHv4562vopiMLRzAfBgNVHSMEGDAWgBT8hgEclgZe9TLrHv4562vopiMLRzAPBgNV\nHRMBAf8EBTADAQH/MAoGCCqGSM49BAMCA0gAMEUCIQCt9V5F5yY5uviwxGmWihyE\nlUx0YTiBoeK1TbA0l4fwIwIgGWjhhJ5cJ7GC5Ney2heVkIgPWkZ0RKhwvi/f81Uw\nreY=\n-----END CERTIFICATE-----\n" +server: "localhost:4443" +client_label: "" +filesystem_handler: < + configuration_directory: "/grr/src/devenv/config/fleetspeak-client" + state_file: "/grr/persist/fleetspeak-client.state" +> +streaming: true diff --git a/devenv/config/fleetspeak-client/textservices/grr.service b/devenv/config/fleetspeak-client/textservices/grr.service new file mode 100644 index 0000000000..e3ecd162ec --- /dev/null +++ b/devenv/config/fleetspeak-client/textservices/grr.service @@ -0,0 +1,11 @@ +name: "GRR" +factory: "Daemon" +config: { + [type.googleapis.com/fleetspeak.daemonservice.Config]: { + argv: "/grr/persist/venv/bin/python" + argv: "-m" + argv: "grr_response_client.client" + argv: "--config" + argv: "/grr/src/devenv/config/grr-client.yaml" + } +} diff --git a/devenv/config/grr-client.yaml b/devenv/config/grr-client.yaml new file mode 100644 index 0000000000..12a3ccae86 --- /dev/null +++ b/devenv/config/grr-client.yaml @@ -0,0 +1,7 @@ +Client.fleetspeak_enabled: true +Client.foreman_check_frequency: 30 +Logging.verbose: true +Logging.engines: file,stderr +Logging.path: /grr +Logging.filename: /grr/grr-client.log + diff --git a/devenv/config/grr-server.yaml b/devenv/config/grr-server.yaml new file mode 100644 index 0000000000..86e87975d3 --- /dev/null +++ b/devenv/config/grr-server.yaml @@ -0,0 +1,63 @@ +Database.implementation: MysqlDB +Blobstore.implementation: DbBlobStore + +Mysql.host: 127.0.0.1 +Mysql.port: 3306 +Mysql.database_name: grr +Mysql.database: grr +Mysql.database_username: grr +Mysql.username: grr +Mysql.database_password: grr +Mysql.password: grr + +Client.server_urls: +- http://localhost:8080/ + +AdminUI.url: http://localhost:8000 +AdminUI.bind: 0.0.0.0 +AdminUI.headless: false +AdminUI.webauth_manager: BasicWebAuthManager +AdminUI.use_precompiled_js: True +AdminUI.csrf_secret_key: O$YTx01F2BuxgT4DoVSI hJVuhWOxyuuxJhg#lg96ARF_fWLabzN5kjAy0UBcI9GDrUPo1v + b7PyF3vj4pqRrc8oegeJ8XgIyI5X + +API.DefaultRouter: ApiCallRouterWithoutChecks + +Logging.domain: localhost +Logging.verbose: true +Logging.engines: file +Logging.path: /grr +Logging.filename: /grr/grr-server.log + +Monitoring.alert_email: grr-monitoring@localhost +Monitoring.emergency_access_email: grr-emergency@localhost + +Client.fleetspeak_enabled: true + +ClientBuilder.fleetspeak_bundled: true + +FleetspeakFrontend Context: + Server.fleetspeak_message_listen_address: 0.0.0.0:11111 + +Server.fleetspeak_enabled: true +Server.fleetspeak_server: localhost:4444 +Server.initialized: true + +Frontend.certificate: | + -----BEGIN CERTIFICATE----- + MIICuTCCAaGgAwIBAgIBAjANBgkqhkiG9w0BAQsFADAgMREwDwYDVQQDDAhncnJf + dGVzdDELMAkGA1UEBhMCVVMwHhcNMjMwMTIyMTEyMTA2WhcNMzMwMTIwMTEyMTA2 + WjAOMQwwCgYDVQQDDANncnIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB + AQDRgX3/3lvGJ2wHO502LFBmNOdN3OHqeo8LNpam0wzDYKevZUpebcCl4aiqYU8g + t/Cd+F5TCOnjLHRore7c86yzI0cfk2ytP0bTCQsCR6AUXzlSt87J6x510wGgW5oB + pEfdTsBHl+bAm3dzJNA0TzNr2i4VfpV9/L1wEw+Se6lC/J74W+Tjm4cHFtIQcwGt + 547wBU3CN71XFMrV8LhaIT7FV4jOqiGZLCTSSR0143d9TOeEErxwXyqPMPhIF0Xm + ihdd9h6VHq/1L6B0qiKTsGnxdtb0KmBIgs/b9i33PEyAWnTTIcw1eK1ryAulCXjF + e7BDiOtVbz43AjSm1iwaWGM9AgMBAAGjEDAOMAwGA1UdEwEB/wQCMAAwDQYJKoZI + hvcNAQELBQADggEBAIt3sLvzluPDkWvNoDKnil9HQ8zBlP1sxMlwtCvTDZbIiTuM + IK+VL1KuNzGEhpeEbziSpN7ZDUT053xpPYnoZZgQlgLBiNmXJaoHOnj+WAewsK0j + vJm7mxLgqdjkXBVyc7jIE/yoZJihygjwDiA3YgvMj/lWZfqU6f57XJERnVDlEUAW + QP2YYDStQZvQuwdn/Lie3PfNTIgwkFRoFcrd4tQGrWhH7/pEfSetgeGJLbW56xSl + GCXRpNm584CHsx3JzkUNgpM6wl+Jc7arcy8uF6bqbQXOFVL2drgzOFWbI+RXS/74 + TGQFaywDGAmHCMx/vcLacmwycH8tEWVLFP1DbLo= + -----END CERTIFICATE----- diff --git a/devenv/devenv.sh b/devenv/devenv.sh new file mode 100755 index 0000000000..2b85d901a4 --- /dev/null +++ b/devenv/devenv.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +cd "$(dirname "$0")" +python -c "import src; src.main()" "$@" diff --git a/devenv/src/__init__.py b/devenv/src/__init__.py new file mode 100644 index 0000000000..2fc8f8fbf0 --- /dev/null +++ b/devenv/src/__init__.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +"""GRR development environment.""" + +import shutil +import subprocess +import sys +import traceback + +from . import cli +from . import commands +from . import util + + +def main() -> None: + if len(sys.argv) < 2: + sys.argv.append("-h") + + term_cols, _ = shutil.get_terminal_size(fallback=(80, 1)) + args = cli.parse_args() + try: + args.func(args) + except subprocess.CalledProcessError as exc: + fail_lines: list[str] = [ + "subprocess error", + util.str_mid_pad(" COMMAND ", term_cols, "="), + str(exc.cmd), + ] + if exc.stderr: + fail_lines.extend([ + util.str_mid_pad(" STDERR ", term_cols, "="), + exc.stderr.decode("utf-8"), + ]) + util.say_fail("\n".join(fail_lines)) + if exc.stdout: + sys.stderr.write(util.str_mid_pad(" STDOUT ", term_cols, "=") + "\n") + sys.stderr.write(exc.stdout.decode("utf-8")) + sys.stderr.write(util.str_mid_pad(" TRACEBACK ", term_cols, "=") + "\n") + traceback.print_tb(exc.__traceback__, file=sys.stderr) + except Exception as exc: # pylint: disable=broad-exception-caught + util.say_fail(f"{exc}") + sys.stderr.write(util.str_mid_pad(" TRACEBACK ", term_cols, "=") + "\n") + traceback.print_tb(exc.__traceback__, file=sys.stderr) + sys.exit(1) diff --git a/devenv/src/cli.py b/devenv/src/cli.py new file mode 100644 index 0000000000..def3dca9ac --- /dev/null +++ b/devenv/src/cli.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +"""CLI utils.""" + +import argparse +from typing import Any, Callable, Optional + + +_ARG_PARSER = argparse.ArgumentParser(prog="devtool.sh") +_ARG_SUBPARSERS = _ARG_PARSER.add_subparsers() + + +def subcommand( + help: str, # pylint: disable=redefined-builtin + args: Optional[dict[str, dict[str, Any]]] = None, +) -> Callable[[Callable[[argparse.Namespace], None]], Any]: + """Subcommand decorator. + + Devenv subcommands are defined as functions decorated by this decorator. + CLI parsing is handled via the standard argparse Python module. As such, the + parameters of this decorator are passed through to argparse. + + Args: + help: Subcommand help text. This will be passed to add_parser(). + args: A dictionary of subcommand arguments. Each item (key, value) in this + dictionary will generate a call to add_argument(key, **value). + + Returns: + The decorated subcommand function. + """ + args = args or {} + + def decorator(cmd_fn: Callable[[argparse.Namespace], None]) -> Any: + parser = _ARG_SUBPARSERS.add_parser(cmd_fn.__name__, help=help) + parser.set_defaults(func=cmd_fn) + for name, params in args.items(): # type: ignore + parser.add_argument(name, **params) # type: ignore + return cmd_fn + + return decorator + + +def parse_args() -> argparse.Namespace: + return _ARG_PARSER.parse_args() diff --git a/devenv/src/commands.py b/devenv/src/commands.py new file mode 100644 index 0000000000..4813c16ebf --- /dev/null +++ b/devenv/src/commands.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python +"""Devenv subcommands.""" + +# This module contains only CLI subcommands, documented via the cli.subcommand() +# decorator. This makes function docstrings superflous. +# +# pylint: disable=missing-function-docstring + +import argparse +import shutil +import subprocess +from typing import List +import uuid + +from . import cli +from . import config +from . import resdefs +from . import reslib +from . import util +from .util import term + + +class SubcommandError(Exception): + """Catch-all exception for anything subcommand-related.""" + + +@cli.subcommand( + help=( + "Check if the dependencies needed by the GRR devenv are set up on" + " your system." + ) +) +def check_deps(args: argparse.Namespace) -> None: + del args # not used + + util.say("Checking if podman is available ...") + podman = shutil.which("podman") + if not podman: + raise SubcommandError("podman not found") + + util.say("Checking if podman can run a rootless container ...") + proc = subprocess.run( + [ + "podman", + "container", + "run", + "--rm", + "alpine:latest", + "sh", + "-c", + "echo -n foo", + ], + check=False, + capture_output=True, + ) + if proc.returncode != 0 or proc.stdout.decode("utf-8") != "foo": + raise SubcommandError(""" + Error executing podman container. + + Hint: make sure subuids are set up for your user. E.g.: + + $ sudo usermod --add-subuids 500000-565535 --add-subgids 500000-565535 $USER + """) + + util.say("Done. Everything looks OK.") + + +@cli.subcommand(help="Remove all local side-effects of running the GRR devenv.") +def clean_all(args: argparse.Namespace) -> None: + del args # not used + + util.say("Cleaning up ...") + resdefs.DEVENV.deep_clean() + + +@cli.subcommand( + help=( + "Rebuild all GRR components. Usually needed after sizeable source" + " changes." + ) +) +def rebuild_grr(args: argparse.Namespace) -> None: + del args # not used + + grr_components: List[str] = [ + "grr/proto", + "grr/core", + "grr/client", + "grr/client_builder", + "api_client/python", + "grr/server", + ] + builder_ctr = reslib.Container( + name="grr-builder", + image=resdefs.GRR_IMG, + volumes=[resdefs.GRR_SRC_VOL, resdefs.GRR_PERSIST_VOL], + daemonize=False, + command=" ".join([ + f". {resdefs.GRR_PERSIST_VOL.mountpoint}/venv/bin/activate", + f"&& cd {resdefs.GRR_SRC_VOL.mountpoint}", + *[f"&& pip install -e {comp}" for comp in grr_components], + ]), + ) + if builder_ctr.is_up(): + raise SubcommandError("A build is already in progress.") + with reslib.cleaner([builder_ctr]): + builder_ctr.create() + + +# pylint: disable=use-dict-literal +@cli.subcommand( + help="Restart one of the GRR components.", + args={ + "-a": dict( + help="Attach to the container TTY after restart.", + default=False, + action="store_true", + ), + "container": dict( + choices=[ctr.name for ctr in resdefs.DEVENV.containers()], + action="store", + ), + }, +) +def restart(args: argparse.Namespace) -> None: + ctr = resdefs.DEVENV.container_by_name(args.container) + if ctr is None: + raise SubcommandError(f"Container not found: {args.container}") + util.say(f"Restarting Container.{args.container}") + if args.a: + detach_keys = config.get("cli.container_detach_keys") + util.say(f"Use {detach_keys} to detach from container tty.") + ctr.restart(attach=args.a) + + +@cli.subcommand( + help="Open a shell in a GRR container.", + args={ + "-c": dict( + help=( + "Open the shell inside this running container." + " If this option is not specified, the shell will be opened in" + " a new container, prepared to run any GRR component." + ), + choices=[ctr.name for ctr in resdefs.DEVENV.containers()], + action="store", + ) + }, +) +def shell(args: argparse.Namespace) -> None: + if args.c: + ctr = resdefs.DEVENV.container_by_name(args.c) + if ctr is None: + raise SubcommandError(f"Container not found: {args.c}") + if not ctr.is_up(): + raise SubcommandError(f"Container {args.c} is not running") + util.say(f"Opening shell inside Container.{args.c} ...") + subprocess.run( + ["podman", "container", "exec", "-it", args.c, "bash"], check=True + ) + else: + ctr_name = f"grr-shell-{str(uuid.uuid4())[:8]}" + ctr = reslib.Container( + name=ctr_name, + image=resdefs.GRR_IMG, + volumes=[resdefs.GRR_SRC_VOL, resdefs.GRR_PERSIST_VOL], + pod=resdefs.GRR_POD, + command="bash", + daemonize=False, + ) + ctr.ensure() + + +@cli.subcommand(help="Start the GRR dev environment.") +def start(args: argparse.Namespace) -> None: + del args # not used + + util.say("Starting devenv ...") + resdefs.DEVENV.ensure() + util.say(f""" + Devenv is now running inside podman pod {term.attn(resdefs.GRR_POD.name)}. + + Admin UI is available at http://localhost:{config.get("net.admin_ui_port")} + user: {config.get("ui.admin_user")} + password: {config.get("ui.admin_password")} + + MySQL raw access is available at localhost:{config.get("net.mysql_port")}. + user: grr + password: grr + """) + + +@cli.subcommand(help="Show status of all devenv resources.") +def status(args: argparse.Namespace) -> None: + del args # not used + + memo: dict[str, bool] = {} + + def res_key(res: reslib.Resource) -> str: + return f"{res.__class__.__name__}.{res.name}" + + def walk(res: reslib.Resource, indent: str) -> None: + key = res_key(res) + deco = term.meh + if key not in memo: + memo[key] = res.is_up() + deco = term.ok if memo[key] else term.fail + tag = "o" if memo[key] else "x" + print(deco(f"{indent}{tag} {res_key(res)}")) + for dep in res.deps: + walk(dep, indent + " ") + + walk(resdefs.DEVENV, "") + + +@cli.subcommand(help="Stop the GRR dev environment.") +def stop(args: argparse.Namespace) -> None: + del args # not used + + util.say("Stopping devenv ...") + resdefs.DEVENV.destroy() diff --git a/devenv/src/config.py b/devenv/src/config.py new file mode 100644 index 0000000000..37acd950d3 --- /dev/null +++ b/devenv/src/config.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +"""Config vars.""" + +import pathlib +from typing import Any, Dict + + +CONFIG: Dict[str, Any] = {} +CONFIG["path.src_dir"] = pathlib.Path(__file__).parent.parent.parent.resolve() +CONFIG["path.containers_dir"] = ( + CONFIG["path.src_dir"] + .joinpath("devenv") + .joinpath("src") + .joinpath("containers") +) +CONFIG["net.admin_ui_port"] = 4280 +CONFIG["net.mysql_port"] = 4236 +CONFIG["ui.admin_user"] = "admin" +CONFIG["ui.admin_password"] = "admin" +CONFIG["build.nodejs_version"] = "16.13.0" +CONFIG["cli.container_detach_keys"] = "ctrl-p,ctrl-q" + + +def get(key: str) -> Any: + return CONFIG[key] diff --git a/devenv/src/containers/fleetspeak-server/Containerfile b/devenv/src/containers/fleetspeak-server/Containerfile new file mode 100644 index 0000000000..5d2d636457 --- /dev/null +++ b/devenv/src/containers/fleetspeak-server/Containerfile @@ -0,0 +1,18 @@ +FROM ubuntu:22.04 as builder + +RUN apt-get update +RUN apt-get install -y python3-pip +RUN pip install fleetspeak-server-bin + + +FROM ubuntu:22.04 + +RUN mkdir -p /fleetspeak +COPY --from=builder \ + /usr/local/fleetspeak-server-bin/usr/bin/fleetspeak-server \ + /fleetspeak/fleetspeak-server +COPY config /fleetspeak/config +COPY run.sh /fleetspeak/run.sh + +WORKDIR /fleetspeak +CMD ["/fleetspeak/run", "admin"] diff --git a/devenv/src/containers/fleetspeak-server/config/admin.components.config b/devenv/src/containers/fleetspeak-server/config/admin.components.config new file mode 100644 index 0000000000..2a260d828f --- /dev/null +++ b/devenv/src/containers/fleetspeak-server/config/admin.components.config @@ -0,0 +1,5 @@ +mysql_data_source_name: "fleetspeak:fleetspeak@tcp(localhost:3306)/fleetspeak" +admin_config: < + listen_address: ":4444" +> +notification_use_http_notifier: true diff --git a/devenv/src/containers/fleetspeak-server/config/frontend.components.config b/devenv/src/containers/fleetspeak-server/config/frontend.components.config new file mode 100644 index 0000000000..973b954b5a --- /dev/null +++ b/devenv/src/containers/fleetspeak-server/config/frontend.components.config @@ -0,0 +1,8 @@ +mysql_data_source_name: "fleetspeak:fleetspeak@tcp(localhost:3306)/fleetspeak" +https_config: < + listen_address: ":4443" + certificates: "-----BEGIN CERTIFICATE-----\nMIIBrjCCAVSgAwIBAgIUS8640c9otFFdzIXYRJ5iQ6qXypswCgYIKoZIzj0EAwIw\nFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTIzMDQyNzA4MjkxOFoXDTMzMDQyNDA4\nMjkxOFowFDESMBAGA1UEAwwJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0D\nAQcDQgAE0i/hPNwkQd9AdtCz7Idh38K1AefNjHrzIkgn+eQM/Z6+3AULd+ngYsPR\nfdkPjSRtxQ5TwL+oDddAJORi6r4Et6OBgzCBgDAtBgNVHREEJjAkghdzdmMtZmxl\nZXRzcGVhay1mcm9udGVuZIIJbG9jYWxob3N0MB0GA1UdDgQWBBT8hgEclgZe9TLr\nHv4562vopiMLRzAfBgNVHSMEGDAWgBT8hgEclgZe9TLrHv4562vopiMLRzAPBgNV\nHRMBAf8EBTADAQH/MAoGCCqGSM49BAMCA0gAMEUCIQCt9V5F5yY5uviwxGmWihyE\nlUx0YTiBoeK1TbA0l4fwIwIgGWjhhJ5cJ7GC5Ney2heVkIgPWkZ0RKhwvi/f81Uw\nreY=\n-----END CERTIFICATE-----\n" + key: "-----BEGIN EC PARAMETERS-----\nBggqhkjOPQMBBw==\n-----END EC PARAMETERS-----\n-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIKM1iz2bLhEFVbjnoDy4sxinn5OHyk75WpqWFpBScAmpoAoGCCqGSM49\nAwEHoUQDQgAE0i/hPNwkQd9AdtCz7Idh38K1AefNjHrzIkgn+eQM/Z6+3AULd+ng\nYsPRfdkPjSRtxQ5TwL+oDddAJORi6r4Etw==\n-----END EC PRIVATE KEY-----\n" +> +notification_listen_address: ":4445" +notification_public_address: "%PUBLIC_IP%:4445" diff --git a/devenv/src/containers/fleetspeak-server/config/services.config b/devenv/src/containers/fleetspeak-server/config/services.config new file mode 100644 index 0000000000..86527795ca --- /dev/null +++ b/devenv/src/containers/fleetspeak-server/config/services.config @@ -0,0 +1,13 @@ +services { + name: "GRR" + factory: "GRPC" + config { + [type.googleapis.com/fleetspeak.grpcservice.Config] { + target: "localhost:11111" + insecure: true + } + } +} +broadcast_poll_time { + seconds: 1 +} diff --git a/devenv/src/containers/fleetspeak-server/run.sh b/devenv/src/containers/fleetspeak-server/run.sh new file mode 100644 index 0000000000..a00403e3b4 --- /dev/null +++ b/devenv/src/containers/fleetspeak-server/run.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +set -e + +THIS_DIR="$(cd "$(dirname "$0")" && pwd)" + +USAGE=" +Fleetspeak server startup script. +Usage: $0 admin|(frontend PUBLIC_IP) +" + +function main() { + local public_ip= + local component= + + case "$1" in + -h|--help) + echo "$USAGE" + exit 1 + ;; + admin) + component=admin + shift + ;; + frontend) + component=frontend + public_ip=$2 + [[ -n "$public_ip" ]] || { + echo "The frontend needs a public IP provided. See --help" + exit 1 + } + sed -i "s/%PUBLIC_IP%/$public_ip/g" \ + "$THIS_DIR/config/frontend.components.config" + shift + ;; + *) + echo "Unknown arg: $1. See --help" + exit 1 + ;; + esac + + "$THIS_DIR/fleetspeak-server" \ + -components_config "$THIS_DIR/config/$component.components.config" \ + -services_config "$THIS_DIR/config/services.config" \ + -logtostderr \ + "$@" +} + +main "$@" diff --git a/devenv/src/containers/grr/Containerfile b/devenv/src/containers/grr/Containerfile new file mode 100644 index 0000000000..2d8c6f91c1 --- /dev/null +++ b/devenv/src/containers/grr/Containerfile @@ -0,0 +1,31 @@ +# Expected build-time mounts: +# - /grr/src +# - /grr/persist + +FROM ubuntu:22.04 + +ARG NODEJS_VERSION + +RUN apt-get update && apt-get install -y \ + python3-pip \ + python-is-python3 \ + python3.10-venv \ + python3-yaml \ + pkg-config \ + libmysqlclient-dev \ + openjdk-19-jre + +RUN cd /grr/persist \ + && python -m venv --system-site-packages venv \ + && . venv/bin/activate \ + && pip install mysqlclient nodeenv wheel \ + && nodeenv -p --prebuilt --node=$NODEJS_VERSION + +RUN cd /grr/src \ + && . /grr/persist/venv/bin/activate \ + && pip install -e grr/proto \ + && pip install -e grr/core \ + && pip install -e grr/client \ + && pip install -e grr/client_builder \ + && pip install -e api_client/python \ + && pip install -e grr/server diff --git a/devenv/src/containers/mysql/Containerfile b/devenv/src/containers/mysql/Containerfile new file mode 100644 index 0000000000..ea12fd0c85 --- /dev/null +++ b/devenv/src/containers/mysql/Containerfile @@ -0,0 +1,18 @@ +# Expected build-time mounts: +# - /var/lib/mysql + +FROM ubuntu:22.04 + +RUN apt-get update && apt-get install -y mysql-server + +COPY mysqld.cnf /etc/mysql/mysql.conf.d/mysqld.cnf + +COPY setup.sql /tmp/setup.sql +RUN service mysql start \ + && cat /tmp/setup.sql | mysql \ + && rm -f /tmp/setup.sql \ + && service mysql stop + +RUN chown -R root:root /var/lib/mysql + +CMD /usr/bin/mysqld_safe diff --git a/devenv/src/containers/mysql/mysqld.cnf b/devenv/src/containers/mysql/mysqld.cnf new file mode 100644 index 0000000000..afcc361e28 --- /dev/null +++ b/devenv/src/containers/mysql/mysqld.cnf @@ -0,0 +1,81 @@ +# +# The MySQL database server configuration file. +# +# One can use all long options that the program supports. +# Run program with --help to get a list of available options and with +# --print-defaults to see which it would actually understand and use. +# +# For explanations see +# http://dev.mysql.com/doc/mysql/en/server-system-variables.html + +# Here is entries for some specific programs +# The following values assume you have at least 32M ram + +[mysqld] +# +# * Basic Settings +# +user = root +# pid-file = /var/run/mysqld/mysqld.pid +# socket = /var/run/mysqld/mysqld.sock +port = 3306 +# datadir = /var/lib/mysql + + +# If MySQL is running as a replication slave, this should be +# changed. Ref https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_tmpdir +# tmpdir = /tmp +# +# Instead of skip-networking the default is now to listen only on +# localhost which is more compatible and is not less secure. +bind-address = 0.0.0.0 +mysqlx-bind-address = 127.0.0.1 +# +# * Fine Tuning +# +key_buffer_size = 16M +max_allowed_packet = 64M +# thread_stack = 256K +innodb_log_file_size = 1G + +# transaction_isolation = READ-COMMITTED + +# thread_cache_size = -1 + +# This replaces the startup script and checks MyISAM tables if needed +# the first time they are touched +myisam-recover-options = BACKUP + +max_connections = 1000 + +# table_open_cache = 4000 + +# +# * Logging and Replication +# +# Both location gets rotated by the cronjob. +# +# Log all queries +# Be aware that this log type is a performance killer. +# general_log_file = /var/log/mysql/query.log +# general_log = 1 +# +# Error log - should be very few entries. +# +log_error = /var/log/mysql/error.log +# +# Here you can see queries with especially long duration +slow_query_log = 1 +slow_query_log_file = /var/log/mysql/mysql-slow.log +# long_query_time = 2 +# log-queries-not-using-indexes +# +# The following can be used as easy to replay backup logs or for replication. +# note: if you are setting up a replication slave, see README.Debian about +# other settings you may need to change. +# server-id = 1 +# log_bin = /var/log/mysql/mysql-bin.log +# binlog_expire_logs_seconds = 2592000 +max_binlog_size = 100M +# binlog_do_db = include_database_name +# binlog_ignore_db = include_database_name diff --git a/devenv/src/containers/mysql/setup.sql b/devenv/src/containers/mysql/setup.sql new file mode 100644 index 0000000000..1a365097d1 --- /dev/null +++ b/devenv/src/containers/mysql/setup.sql @@ -0,0 +1,14 @@ +CREATE DATABASE grr; + +CREATE + USER grr IDENTIFIED BY "grr"; + +GRANT ALL PRIVILEGES ON grr.* TO 'grr' @'%'; + +CREATE DATABASE fleetspeak; + +CREATE + USER fleetspeak IDENTIFIED BY "fleetspeak"; + +GRANT ALL PRIVILEGES ON fleetspeak.* TO 'fleetspeak' @'%'; +GRANT SUPER ON *.* TO 'grr' @'%'; diff --git a/devenv/src/mypy.ini b/devenv/src/mypy.ini new file mode 100644 index 0000000000..685c02599f --- /dev/null +++ b/devenv/src/mypy.ini @@ -0,0 +1,3 @@ +[mypy] +disallow_untyped_defs = True +check_untyped_defs = True diff --git a/devenv/src/pylintrc b/devenv/src/pylintrc new file mode 100644 index 0000000000..2f4ecb6a1e --- /dev/null +++ b/devenv/src/pylintrc @@ -0,0 +1,8 @@ +[FORMAT] + +indent-string=' ' + + +[MESSAGES CONTROL] + +disable=missing-class-docstring, missing-function-docstring diff --git a/devenv/src/pytest.ini b/devenv/src/pytest.ini new file mode 100644 index 0000000000..2864d8308d --- /dev/null +++ b/devenv/src/pytest.ini @@ -0,0 +1 @@ +# devenv pytest.ini diff --git a/devenv/src/resdefs.py b/devenv/src/resdefs.py new file mode 100644 index 0000000000..ed5476e726 --- /dev/null +++ b/devenv/src/resdefs.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python +"""Resource definitions.""" + +from typing import List, Optional + +from . import config +from . import reslib + + +MYSQL_DATA_VOL = reslib.Volume( + name="grr-mysql-data-vol", mountpoint="/var/lib/mysql" +) + +MYSQL_IMG = reslib.LocalImage( + name="grr-mysql-img", + context_dir=config.get("path.containers_dir").joinpath("mysql"), +) + +FLEETSPEAK_SERVER_IMG = reslib.LocalImage( + name="grr-fleetspeak-server-img", + context_dir=config.get("path.containers_dir").joinpath("fleetspeak-server"), +) + +GRR_SRC_VOL = reslib.HostPathVolume( + name=str(config.get("path.src_dir")), mountpoint="/grr/src" +) + +GRR_PERSIST_VOL = reslib.Volume( + name="grr-persist-vol", mountpoint="/grr/persist" +) + +GRR_IMG = reslib.LocalImage( + name="grr-main-img", + context_dir=config.get("path.containers_dir").joinpath("grr"), + build_args={"NODEJS_VERSION": config.get("build.nodejs_version")}, + volumes=[GRR_SRC_VOL, GRR_PERSIST_VOL], +) + +GRR_POD = reslib.Pod( + name="grr-pod", + ports={ + config.get("net.admin_ui_port"): 8000, + config.get("net.mysql_port"): 3306, + }, +) + +MYSQL_CTR = reslib.Container( + name="grr-mysql", + image=MYSQL_IMG, + volumes=[MYSQL_DATA_VOL], + pod=GRR_POD, +) + +FLEETSPEAK_ADMIN_CTR = reslib.Container( + name="grr-fleetspeak-admin", + image=FLEETSPEAK_SERVER_IMG, + pod=GRR_POD, + command="bash /fleetspeak/run.sh admin", + deps=[MYSQL_CTR], +) + +FLEETSPEAK_FRONTEND_CTR = reslib.Container( + name="grr-fleetspeak-frontend", + image=FLEETSPEAK_SERVER_IMG, + pod=GRR_POD, + command="bash /fleetspeak/run.sh frontend localhost", + deps=[MYSQL_CTR], +) + +GRR_ADMIN_UI_CTR = reslib.Container( + name="grr-admin-ui", + image=GRR_IMG, + pod=GRR_POD, + volumes=[GRR_SRC_VOL, GRR_PERSIST_VOL], + command=( + f"{GRR_PERSIST_VOL.mountpoint}/venv/bin/python" + " -m grr_response_server.gui.admin_ui" + f" --config {GRR_SRC_VOL.mountpoint}/devenv/config/grr-server.yaml" + ), + deps=[MYSQL_CTR], +) + +GRR_WORKER_CTR = reslib.Container( + name="grr-worker", + image=GRR_IMG, + pod=GRR_POD, + volumes=[GRR_SRC_VOL, GRR_PERSIST_VOL], + command=( + f"{GRR_PERSIST_VOL.mountpoint}/venv/bin/python" + " -m grr_response_server.bin.worker" + f" --config {GRR_SRC_VOL.mountpoint}/devenv/config/grr-server.yaml" + ), + deps=[MYSQL_CTR], +) + +GRR_FRONTEND_CTR = reslib.Container( + name="grr-frontend", + image=GRR_IMG, + pod=GRR_POD, + volumes=[GRR_SRC_VOL, GRR_PERSIST_VOL], + command=( + f"{GRR_PERSIST_VOL.mountpoint}/venv/bin/python" + " -m grr_response_server.bin.fleetspeak_frontend" + f" --config {GRR_SRC_VOL.mountpoint}/devenv/config/grr-server.yaml" + ), + deps=[MYSQL_CTR], +) + +GRR_CLIENT_CTR = reslib.Container( + name="grr-client", + image=GRR_IMG, + pod=GRR_POD, + volumes=[GRR_SRC_VOL, GRR_PERSIST_VOL], + command=( + f"{GRR_PERSIST_VOL.mountpoint}/venv/fleetspeak-client-bin/usr/bin/fleetspeak-client" + " -config" + f" {GRR_SRC_VOL.mountpoint}/devenv/config/fleetspeak-client/client.config" + ), +) + +ADMIN_USER = reslib.AdminUser( + name=config.get("ui.admin_user"), + password=config.get("ui.admin_password"), + grr_pod=GRR_POD, + grr_img=GRR_IMG, + grr_persist_vol=GRR_PERSIST_VOL, + grr_src_vol=GRR_SRC_VOL, + mysql_ctr=MYSQL_CTR, +) + + +class Devenv(reslib.Resource): + """Wrapper resource for the entire devenv.""" + + components: List[reslib.Resource] = [ + MYSQL_CTR, + FLEETSPEAK_ADMIN_CTR, + FLEETSPEAK_FRONTEND_CTR, + GRR_ADMIN_UI_CTR, + GRR_WORKER_CTR, + GRR_FRONTEND_CTR, + GRR_CLIENT_CTR, + ADMIN_USER, + ] + + def __init__(self) -> None: + super().__init__("grr-devenv", self.components[:]) + + def is_up(self) -> bool: + for comp in self.components: + if not comp.is_up(): + return False + return True + + def create(self) -> None: + pass + + def destroy(self) -> None: + GRR_POD.clean() + + def containers(self) -> List[reslib.Container]: + return [ctr for ctr in self.components if isinstance(ctr, reslib.Container)] + + def container_by_name(self, name: str) -> Optional[reslib.Container]: + for comp in self.components: + if isinstance(comp, reslib.Container) and comp.name == name: + return comp + return None + + +DEVENV = Devenv() diff --git a/devenv/src/reslib.py b/devenv/src/reslib.py new file mode 100644 index 0000000000..5e61c89592 --- /dev/null +++ b/devenv/src/reslib.py @@ -0,0 +1,391 @@ +#!/usr/bin/env python +"""Dev environment resource lib.""" + +import abc +import contextlib +import pathlib +import subprocess +import sys +import traceback +from typing import Any, Dict, Iterable, Iterator, List, Optional + +from . import config +from . import util + + +class ResourceError(Exception): + """Catch-all exception for all resource-related errors.""" + + +class Resource(abc.ABC): + """An abstract resource. + + Concrete resources extend this class by implementing the creation, + destruction, and check member functions. + """ + + def __init__(self, name: str, deps: Optional[List["Resource"]] = None): + self.name: str = name + self.deps: List["Resource"] = deps or [] + + @abc.abstractmethod + def create(self) -> None: + pass + + @abc.abstractmethod + def destroy(self) -> None: + pass + + @abc.abstractmethod + def is_up(self) -> bool: + pass + + def ensure(self) -> None: + if self.is_up(): + return + for dep in self.deps: + dep.ensure() + util.say(f"Creating {self.__class__.__name__}.{self.name} ...") + self.create() + + def clean(self) -> None: + if self.is_up(): + self.destroy() + + def deep_clean(self) -> None: + self.clean() + for dep in self.deps: + dep.deep_clean() + + +@contextlib.contextmanager +def cleaner(resources: Iterable[Resource]) -> Iterator[None]: + try: + yield None + finally: + for res in resources: + try: + res.clean() + except Exception: # pylint: disable=broad-exception-caught + util.say_fail(f"Error cleaning up {res.__class__.__name__}.{res.name}") + traceback.print_exc(file=sys.stderr) + + +class Volume(Resource): + """Container volume.""" + + def __init__( + self, name: str, mountpoint: str, deps: Optional[List[Resource]] = None + ) -> None: + super().__init__(name, deps) + self.mountpoint = mountpoint + + def is_up(self) -> bool: + proc = subprocess.run( + f"podman volume exists {self.name}", + shell=True, + check=False, + capture_output=True, + ) + return proc.returncode == 0 + + def create(self) -> None: + subprocess.run( + f"podman volume create {self.name}", + shell=True, + check=True, + capture_output=True, + ) + + def destroy(self) -> None: + subprocess.run( + f"podman volume rm {self.name}", + shell=True, + check=True, + capture_output=True, + ) + + @property + def host_path(self) -> pathlib.Path: + proc = subprocess.run( + f'podman volume inspect --format "{{{{.Mountpoint}}}}" {self.name}', + shell=True, + check=True, + capture_output=True, + ) + return pathlib.Path(proc.stdout.decode("utf-8").strip()) + + +class HostPathVolume(Volume): + """Container volume backed by a host directory.""" + + def is_up(self) -> bool: + return self.host_path.is_dir() + + def create(self) -> None: + raise ResourceError("Attempted to use HostPathVolume without a host path.") + + def destroy(self) -> None: + pass + + @property + def host_path(self) -> pathlib.Path: + return pathlib.Path(self.name) + + +class Image(Resource): + """A container image.""" + + def __init__(self, *args: Any, **kwargs: Any): + super().__init__(*args, **kwargs) + self._pulled: bool = False + + def is_up(self) -> bool: + proc = subprocess.run( + f"podman image exists {self.name}", + shell=True, + check=False, + capture_output=True, + ) + return proc.returncode == 0 + + def create(self) -> None: + subprocess.run( + f"podman image pull {self.name}", + shell=True, + check=True, + capture_output=True, + ) + self._pulled = True + + def destroy(self) -> None: + if self._pulled: + subprocess.run( + f"podman image rm -f {self.name}", + shell=True, + check=True, + capture_output=True, + ) + + +class LocalImage(Image): + """A locally-created container image.""" + + # pylint: disable=too-many-arguments + def __init__( + self, + name: str, + context_dir: pathlib.Path, + build_args: Optional[dict[str, str]] = None, + volumes: Optional[List[Volume]] = None, + deps: Optional[List[Resource]] = None, + ) -> None: + volumes = volumes or [] + build_args = build_args or {} + deps = deps or [] + deps.extend(volumes) + super().__init__(name, deps) + + self.context_dir: pathlib.Path = context_dir + self.build_args: dict[str, str] = build_args + self.volumes: List[Volume] = volumes + + def create(self) -> None: + cmdln = ["podman", "image", "build", "--no-cache", "-t", self.name] + for key, val in self.build_args.items(): + cmdln.extend(["--build-arg", f"{key}={val}"]) + for vol in self.volumes: + cmdln.extend(["--volume", f"{vol.host_path}:{vol.mountpoint}"]) + cmdln.append(str(self.context_dir)) + subprocess.run(cmdln, check=True, capture_output=True) + + def destroy(self) -> None: + subprocess.run( + f"podman image rm -f {self.name}", + shell=True, + check=True, + capture_output=True, + ) + + +class Pod(Resource): + """A podman pod.""" + + def __init__( + self, + name: str, + ports: Optional[Dict[int, int]] = None, + deps: Optional[List[Resource]] = None, + ): + super().__init__(name, deps) + self.ports: Dict[int, int] = ports or {} + + def is_up(self) -> bool: + proc = subprocess.run( + f"podman pod exists {self.name}", + shell=True, + check=False, + capture_output=True, + ) + return proc.returncode == 0 + + def create(self) -> None: + port_args = " ".join([f"-p {hp}:{cp}" for hp, cp in self.ports.items()]) + subprocess.run( + f"podman pod create {port_args} --name {self.name}", + shell=True, + check=True, + capture_output=True, + ) + + def destroy(self) -> None: + subprocess.run( + f"podman pod rm -f {self.name}", + shell=True, + check=True, + capture_output=True, + ) + + +class Container(Resource): + """A (podman) container.""" + + # pylint: disable=too-many-arguments + def __init__( + self, + name: str, + image: Image, + volumes: Optional[List[Volume]] = None, + command: Optional[str] = None, + pod: Optional[Pod] = None, + daemonize: bool = True, + deps: Optional[List[Resource]] = None, + ): + volumes = volumes or [] + deps = deps or [] + deps.append(image) + deps.extend(volumes) + if pod: + deps.append(pod) + super().__init__(name, deps) + + self.image: Image = image + self.volumes: List[Volume] = volumes + self.pod: Optional[Pod] = pod + self.command: Optional[str] = command + self.daemonize = daemonize + + def is_up(self) -> bool: + proc = subprocess.run( + f"podman container inspect --format {{{{.State.Status}}}} {self.name}", + shell=True, + check=False, + capture_output=True, + ) + status = proc.stdout.decode("utf-8").strip() + return proc.returncode == 0 and status == "running" + + def create(self) -> None: + subprocess.run( + f"podman container rm -f {self.name}", + shell=True, + check=False, + capture_output=True, + ) + + cmdln = ["podman", "container", "run", "--rm", "-i", "--name", self.name] + if self.pod: + cmdln.extend(["--pod", self.pod.name]) + if self.daemonize: + cmdln.extend(["--detach", "-t", "--init", "--restart=on-failure"]) + elif sys.stdout.isatty(): + cmdln.extend([ + "-t", + f"""--detach-keys={config.get("cli.container_detach_keys")}""", + ]) + for vol in self.volumes: + cmdln.extend(["--volume", f"{vol.name}:{vol.mountpoint}"]) + cmdln.append(self.image.name) + if self.command: + cmdln.extend(["sh", "-c", self.command]) + subprocess.run(cmdln, check=True, capture_output=self.daemonize) + + def destroy(self) -> None: + subprocess.run( + f"podman container rm -f {self.name}", + shell=True, + check=True, + capture_output=True, + ) + + def restart(self, attach: bool = False) -> None: + if self.is_up(): + self.destroy() + self.daemonize = not attach + self.ensure() + + +class AdminUser(Resource): + """GRR admin user. Used to login to AdminUI.""" + + # pylint: disable=too-many-arguments + def __init__( + self, + name: str, + password: str, + grr_pod: Pod, + grr_img: LocalImage, + grr_persist_vol: Volume, + grr_src_vol: Volume, + mysql_ctr: Container, + ) -> None: + deps = [grr_img, grr_persist_vol, mysql_ctr, grr_pod] + super().__init__(name, deps) + self.password: str = password + + self.grr_pod: Pod = grr_pod + self.grr_img: LocalImage = grr_img + self.grr_persist_vol: Volume = grr_persist_vol + self.grr_src_vol: Volume = grr_src_vol + self.mysql_ctr: Container = mysql_ctr + + def is_up(self) -> bool: + if not self.mysql_ctr.is_up(): + return False + query = f'SELECT COUNT(*) FROM grr.grr_users WHERE username="{self.name}"' + proc = subprocess.run( + ["podman", "exec", self.mysql_ctr.name, "mysql", "-N", "-e", query], + check=False, + capture_output=True, + ) + count = proc.stdout.decode("utf-8").strip() + return proc.returncode == 0 and count != "0" + + def create(self) -> None: + subprocess.run( + [ + "podman", + "run", + "--rm", + "-v", + f"{self.grr_persist_vol.name}:{self.grr_persist_vol.mountpoint}", + "-v", + f"{self.grr_src_vol.name}:{self.grr_src_vol.mountpoint}", + "--pod", + self.grr_pod.name, + self.grr_img.name, + f"{self.grr_persist_vol.mountpoint}/venv/bin/grr_config_updater", + "--config", + f"{self.grr_src_vol.mountpoint}/devenv/config/grr-server.yaml", + "add_user", + self.name, + "--password", + self.password, + "--admin", + "true", + ], + check=True, + ) + + def destroy(self) -> None: + pass diff --git a/devenv/src/reslib_test.py b/devenv/src/reslib_test.py new file mode 100644 index 0000000000..2d16dbb7c8 --- /dev/null +++ b/devenv/src/reslib_test.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +"""Resource tests.""" + +import pathlib +import tempfile +import uuid + +from . import reslib + + +def _temp_name(prefix: str = "") -> str: + return f"{prefix}-{uuid.uuid4()}" + + +class TestRes: + + def test_volume(self) -> None: + vol = reslib.Volume(_temp_name("test-volume"), mountpoint="/foo") + with reslib.cleaner((vol,)): + assert not vol.is_up() + vol.create() + assert vol.is_up() + assert vol.host_path.is_dir() + assert not vol.is_up() + + def test_local_image(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir_str: + temp_dir = pathlib.Path(temp_dir_str) + img = reslib.LocalImage( + name=_temp_name("test-image"), + context_dir=temp_dir, + ) + with open( + temp_dir.joinpath("Containerfile"), "w", encoding="utf-8" + ) as ctrf: + ctrf.write("\n".join(["FROM alpine:latest", "RUN echo bar > /foo"])) + ctr = reslib.Container( + name=_temp_name("test-container"), + image=img, + daemonize=False, + command="[ $(cat /foo) = bar ]", + ) + with reslib.cleaner([ctr, img]): + ctr.ensure() + + def test_container(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + vol = reslib.HostPathVolume(str(temp_dir), mountpoint="/data") + img = reslib.Image(name="alpine:latest") + ctr = reslib.Container( + name=_temp_name("test-container"), + image=img, + volumes=[vol], + command="echo -n bar > /data/foo", + daemonize=False, + ) + with reslib.cleaner((ctr, vol, img)): + ctr.ensure() + with open( + pathlib.Path(temp_dir).joinpath("foo"), mode="r", encoding="utf-8" + ) as foof: + assert foof.read() == "bar" diff --git a/devenv/src/util/__init__.py b/devenv/src/util/__init__.py new file mode 100644 index 0000000000..57914fc724 --- /dev/null +++ b/devenv/src/util/__init__.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python +"""Misc utils (aka #include ).""" + +import sys + +from .. import config +from . import term + + +_TAG = "[grr-devenv]" + + +def say(msg: str) -> None: + tag: str = term.ok(_TAG) + sys.stdout.write(f"{tag} {msg}\n") + + +def say_fail(msg: str) -> None: + buf: str = term.fail(f"{_TAG} {msg}") + sys.stderr.write(buf + "\n") + + +def say_warn(msg: str) -> None: + buf: str = term.warn(f"{_TAG} {msg}") + sys.stderr.write(buf + "\n") + + +# pylint: disable=invalid-name +def str_mid_pad(s: str, width: int, fill: str) -> str: + pad = fill * int((width - len(s)) / (2 * len(fill))) + return f"{pad}{s}{pad}" diff --git a/devenv/src/util/term.py b/devenv/src/util/term.py new file mode 100644 index 0000000000..f1201171e8 --- /dev/null +++ b/devenv/src/util/term.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +"""Terminal pretty stuffs.""" + +import sys +from typing import Callable + + +_ESC = "\x1b" +_RESET = f"{_ESC}[0m" +_RED_FG = f"{_ESC}[1;31m" +_GREEN_FG = f"{_ESC}[1;32m" +_YELLOW_FG = f"{_ESC}[1;33m" +_GRAY_FG = f"{_ESC}[38;5;240m" +_WHITE_FG = f"{_ESC}[1;37m" + + +def _colorize(buf: str, color_code: str) -> str: + if not sys.stdout.isatty(): + return buf + return f"{color_code}{buf}{_RESET}" + + +fail: Callable[[str], str] = lambda buf: _colorize(buf, _RED_FG) +warn: Callable[[str], str] = lambda buf: _colorize(buf, _YELLOW_FG) +ok: Callable[[str], str] = lambda buf: _colorize(buf, _GREEN_FG) +meh: Callable[[str], str] = lambda buf: _colorize(buf, _GRAY_FG) +attn: Callable[[str], str] = lambda buf: _colorize(buf, _WHITE_FG) diff --git a/grr/client/grr_response_client/actions.py b/grr/client/grr_response_client/actions.py index c8a796ad54..29b4554e14 100644 --- a/grr/client/grr_response_client/actions.py +++ b/grr/client/grr_response_client/actions.py @@ -299,6 +299,16 @@ def SendReply(self, task_id=self.message.Get("task_id") or None, require_fastpoll=self.require_fastpoll) + # If client actions sends replies, it's not dead, thus we should send + # a heartbeat to Fleetspeak. This rule not apply to status messages, + # as they're only sent at the end of the client action processing and + # might, among other things, report a resource-usage-exceeding-limits + # errors that are triggered by self.Progress calls (for such cases + # a self.Progress call here will fail, as by this point the client + # action is already out of resources). + if message_type != rdf_flows.GrrMessage.Type.STATUS: + self.Progress() + def Progress(self): """Indicate progress of the client action. diff --git a/grr/client/grr_response_client/client_actions/action_test.py b/grr/client/grr_response_client/client_actions/action_test.py index 7881d26217..a3c4b2632e 100644 --- a/grr/client/grr_response_client/client_actions/action_test.py +++ b/grr/client/grr_response_client/client_actions/action_test.py @@ -214,7 +214,14 @@ class FakeProcess(object): def __init__(self, pid=None): del pid # unused - self.times = [(1, 0), (2, 0), (3, 0), (10000, 0), (10001, 0)] + self.times = [ + (1, 0), + (2, 0), + (3, 0), + (10000, 0), + (10001, 0), + (10002, 0), + ] self.pcputimes = collections.namedtuple("pcputimes", ["user", "system"]) def cpu_times(self): # pylint: disable=g-bad-name diff --git a/grr/client/grr_response_client/client_actions/admin.py b/grr/client/grr_response_client/client_actions/admin.py index 1f3ffa51f0..660434d642 100644 --- a/grr/client/grr_response_client/client_actions/admin.py +++ b/grr/client/grr_response_client/client_actions/admin.py @@ -15,7 +15,7 @@ import yara from grr_response_client import actions -from grr_response_client import communicator +from grr_response_client import client_metrics from grr_response_client.client_actions import tempfiles from grr_response_client.client_actions import timeline from grr_response_client.unprivileged import sandbox @@ -269,10 +269,11 @@ def Run(self, arg): RSS_size=meminfo.rss, VMS_size=meminfo.vms, memory_percent=proc.memory_percent(), - bytes_received=communicator.GRR_CLIENT_RECEIVED_BYTES.GetValue(), - bytes_sent=communicator.GRR_CLIENT_SENT_BYTES.GetValue(), + bytes_received=client_metrics.GRR_CLIENT_RECEIVED_BYTES.GetValue(), + bytes_sent=client_metrics.GRR_CLIENT_SENT_BYTES.GetValue(), create_time=create_time, - boot_time=boot_time) + boot_time=boot_time, + ) response.cpu_samples = self.grr_worker.stats_collector.CpuSamplesBetween( start_time=arg.start_time, end_time=arg.end_time) diff --git a/grr/client/grr_response_client/client_actions/admin_test.py b/grr/client/grr_response_client/client_actions/admin_test.py index e48c4a020c..1a63dc9c89 100644 --- a/grr/client/grr_response_client/client_actions/admin_test.py +++ b/grr/client/grr_response_client/client_actions/admin_test.py @@ -9,11 +9,9 @@ from absl import app from absl.testing import absltest import psutil -import requests +from grr_response_client import client_metrics from grr_response_client import client_stats -from grr_response_client import comms -from grr_response_client import communicator from grr_response_client.client_actions import admin from grr_response_client.unprivileged import sandbox from grr_response_core import config @@ -69,24 +67,6 @@ def testUpdateConfiguration(self): """ self.assertIn(server_urls, data) - self.urls = [] - - # Now test that our location was actually updated. - - def FakeUrlOpen(url=None, data=None, **_): - self.urls.append(url) - response = requests.Response() - response.status_code = 200 - response._content = data - return response - - with mock.patch.object(requests, "request", FakeUrlOpen): - client_context = comms.GRRHTTPClient(worker_cls=MockClientWorker) - client_context.MakeRequest("") - - # Since the request is successful we only connect to one location. - self.assertIn(location[0], self.urls[0]) - def testOnlyUpdatableFieldsAreUpdated(self): with test_lib.ConfigOverrider({ "Client.server_urls": [u"http://something.com/"], @@ -185,8 +165,8 @@ def setUp(self): def testReturnsAllDataByDefault(self): """Checks that stats collection works.""" - communicator.GRR_CLIENT_RECEIVED_BYTES.Increment(1566) - communicator.GRR_CLIENT_SENT_BYTES.Increment(2000) + client_metrics.GRR_CLIENT_RECEIVED_BYTES.Increment(1566) + client_metrics.GRR_CLIENT_SENT_BYTES.Increment(2000) results = self.RunAction( admin.GetClientStats, diff --git a/grr/client/grr_response_client/client_actions/memory.py b/grr/client/grr_response_client/client_actions/memory.py index c176ffa6d7..65e00aba26 100644 --- a/grr/client/grr_response_client/client_actions/memory.py +++ b/grr/client/grr_response_client/client_actions/memory.py @@ -250,6 +250,8 @@ def Match(self, process, chunks: Iterable[streaming.Chunk], progress: Callable[[], None]) -> Iterator[rdf_memory.YaraMatch]: timeout_secs = (deadline - rdfvalue.RDFDatetime.Now()).ToInt( rdfvalue.SECONDS) + if self._client is None: + raise ValueError("Client not instantiated.") if not self._rules_uploaded: self._client.UploadSignature(self._rules_str) self._rules_uploaded = True @@ -382,6 +384,7 @@ def __init__(self, grr_worker=None): def _ScanRegion( self, process, chunks: Iterable[streaming.Chunk], deadline: rdfvalue.RDFDatetime) -> Iterator[rdf_memory.YaraMatch]: + assert self._yara_wrapper is not None yield from self._yara_wrapper.Match(process, chunks, deadline, self.Progress) diff --git a/grr/client/grr_response_client/client_actions/osquery.py b/grr/client/grr_response_client/client_actions/osquery.py index fc601cc413..3506717efa 100644 --- a/grr/client/grr_response_client/client_actions/osquery.py +++ b/grr/client/grr_response_client/client_actions/osquery.py @@ -212,25 +212,28 @@ def Query(args: rdf_osquery.OsqueryArgs) -> str: "--logger_min_status=3", # Disable status logs. "--logger_min_stderr=2", # Only ERROR-level logs to stderr. "--json", # Set output format to JSON. - args.query, ] proc = subprocess.run( command, timeout=timeout, check=True, + input=args.query, + text=True, + encoding="utf-8", stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + stderr=subprocess.PIPE, + ) except subprocess.TimeoutExpired as error: raise TimeoutError(cause=error) except subprocess.CalledProcessError as error: - stderr = error.stderr.decode("utf-8") + stderr = error.stderr raise Error(message=f"Osquery error on the client: {stderr}") - stderr = proc.stderr.decode("utf-8").strip() + stderr = proc.stderr.strip() if stderr: # Depending on the version, in case of a syntax error osquery might or might # not terminate with a non-zero exit code, but it will always print the # error to stderr. raise Error(message=f"Osquery error on the client: {stderr}") - return proc.stdout.decode("utf-8") + return proc.stdout diff --git a/grr/client/grr_response_client/client_main.py b/grr/client/grr_response_client/client_main.py index 2152ecc2eb..8581bfd9f8 100644 --- a/grr/client/grr_response_client/client_main.py +++ b/grr/client/grr_response_client/client_main.py @@ -1,7 +1,6 @@ #!/usr/bin/env python """This is the entry point for the GRR client.""" -import logging import pdb import platform import sys @@ -11,13 +10,12 @@ from grr_response_client import client_plugins from grr_response_client import client_startup -from grr_response_client import comms from grr_response_client import fleetspeak_client from grr_response_client import installer from grr_response_client.unprivileged import sandbox from grr_response_core import config from grr_response_core.config import contexts -from grr_response_core.lib import config_lib + _INSTALL = flags.DEFINE_bool("install", False, "Specify this to install the client.") @@ -86,31 +84,7 @@ def main(unused_args): config.CONFIG["Source.version_string"]), [config.CONFIG["Client.install_path"]]) - if config.CONFIG["Client.fleetspeak_enabled"]: - fleetspeak_client.GRRFleetspeakClient().Run() - return - - errors = config.CONFIG.Validate(["Client", "CA", "Logging"]) - - if errors and list(errors.keys()) != ["Client.private_key"]: - raise config_lib.ConfigFormatError(errors) - - if config.CONFIG["Client.fleetspeak_enabled"]: - raise ValueError( - "This is not a Fleetspeak client, yet 'Client.fleetspeak_enabled' is " - "set to 'True'.") - - enrollment_necessary = not config.CONFIG.Get("Client.private_key") - # Instantiating the client will create a private_key so we need to use a flag. - client = comms.GRRHTTPClient( - ca_cert=config.CONFIG["CA.certificate"], - private_key=config.CONFIG.Get("Client.private_key", default=None)) - - if enrollment_necessary: - logging.info("No private key found, starting enrollment.") - client.InitiateEnrolment() - - client.Run() + fleetspeak_client.GRRFleetspeakClient().Run() if __name__ == "__main__": app.run(main) diff --git a/grr/client/grr_response_client/client_metrics.py b/grr/client/grr_response_client/client_metrics.py new file mode 100644 index 0000000000..5f16f06e41 --- /dev/null +++ b/grr/client/grr_response_client/client_metrics.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python +"""Metrics used by the client.""" + +from grr_response_core.stats import metrics + +GRR_CLIENT_RECEIVED_BYTES = metrics.Counter("grr_client_received_bytes") +GRR_CLIENT_SENT_BYTES = metrics.Counter("grr_client_sent_bytes") diff --git a/grr/client/grr_response_client/client_plugins.py b/grr/client/grr_response_client/client_plugins.py index e236bedcc6..c466d15954 100644 --- a/grr/client/grr_response_client/client_plugins.py +++ b/grr/client/grr_response_client/client_plugins.py @@ -22,7 +22,6 @@ elif "linux" in sys.platform: from grr_response_client.linux import registry_init as linux_registry_init -from grr_response_client import comms from grr_response_client import local # pylint: enable=g-import-not-at-top,unused-import,g-bad-import-order diff --git a/grr/client/grr_response_client/comms.py b/grr/client/grr_response_client/comms.py index 1e373685e2..89fc5b18c3 100644 --- a/grr/client/grr_response_client/comms.py +++ b/grr/client/grr_response_client/comms.py @@ -1,481 +1,30 @@ #!/usr/bin/env python -"""This class handles the GRR Client Communication. - -The GRR client uses HTTP to communicate with the server. - -The client connections are controlled via a number of config parameters: - -- Client.error_poll_min: Time to wait between retries in an ERROR state. - -- Client.server_urls: A list of URLs for the base control server. - -- Client.proxy_servers: A list of proxies to try to connect through. - -- Client.poll_max, Client.poll_min: Parameters for timing of SLOW POLL and FAST - POLL modes. - -The client goes through a state machine: - -1) In the INITIAL state, the client has no active server URL or active proxy and - it is therefore searching through the list of proxies and connection URLs for - one that works. The client will try each combination of proxy/URL in turn - without delay until a 200 or a 406 message is seen. If all possibilities are - exhausted, and a connection is not established, the client will switch to - SLOW POLL mode (and retry connection every Client.poll_max). - -2) In SLOW POLL mode the client will wait Client.poll_max between re-connection - attempts. - -3) If a server is detected, the client will communicate with it. If the server - returns a 406 error, the client will send an enrollment request. Enrollment - requests are only re-sent every 10 minutes (regardless of the frequency of - 406 responses). Note that a 406 message is considered a valid connection and - the client will not search for URL/Proxy combinations as long as it keep - receiving 406 responses. - -4) During CONNECTED state, the client has a valid server certificate, receives - 200 responses from the server and is able to send messages to the server. The - polling frequency in this state is determined by the polling mode requested - by the messages received or send. If any message from the server or from the - worker queue (to the server) has the require_fastpoll flag set, the client - switches into FAST POLL mode. - -5) When not in FAST POLL mode, the polling frequency is controlled by the - Timer() object. It is currently a geometrically decreasing function which - starts at the Client.poll_min and approaches the Client.poll_max setting. - -6) If a 500 error occurs in the CONNECTED state, the client will assume that the - server is temporarily down. The client will switch to the RETRY state and - retry sending the data with a fixed frequency determined by - Client.error_poll_min to the same URL/Proxy combination. The client will - retry for retry_error_limit times before exiting the - CONNECTED state and returning to the INITIAL state (i.e. the client will - start searching for a new URL/Proxy combination). If a retry is successful, - the client will return to its designated polling frequency. - -7) If there are connection_error_limit failures, the client will - exit. Hopefully the nanny will restart the client. - -Examples: - -1) Client starts up on a disconnected network: Client will try every URL/Proxy - combination once every Client.poll_max (default 10 minutes). - -2) Client connects successful but loses network connectivity. Client will re-try - retry_error_limit (10 times) every Client.error_poll_min (1 Min) to - resent the last message. If it does not succeed it starts searching for a new - URL/Proxy combination as in example 1. - -""" +"""This class handles the GRR Client Communication.""" import collections import logging import os import pdb -import posixpath import queue import signal -import sys import threading import time -import traceback from absl import flags import psutil -import requests from grr_response_client import actions from grr_response_client import client_actions from grr_response_client import client_stats from grr_response_client import client_utils -from grr_response_client import communicator from grr_response_client.client_actions import admin from grr_response_core import config -from grr_response_core.lib import queues from grr_response_core.lib import rdfvalue -from grr_response_core.lib import type_info from grr_response_core.lib import utils -from grr_response_core.lib.rdfvalues import client as rdf_client -from grr_response_core.lib.rdfvalues import crypto as rdf_crypto from grr_response_core.lib.rdfvalues import flows as rdf_flows from grr_response_core.lib.rdfvalues import protodict as rdf_protodict -class HTTPObject(object): - """Data returned from a HTTP connection.""" - - def __init__(self, url="", data="", proxy="", code=500, duration=0): - self.url = url - self.data = data - self.proxy = proxy - self.code = code - # Contains the decoded data from the 'control' endpoint. - self.messages = self.source = self.nonce = None - self.duration = duration - - def Success(self): - """Returns if the request was successful.""" - return self.code in (200, 406) - - -class HTTPManager(object): - """A manager for all HTTP/S connections. - - NOTE: This HTTPManager is not thread safe and should not be shared between - threads. - """ - - # If the client encounters this many connection errors, it searches - # for a new proxy/server url combination. - retry_error_limit = 10 - - # If the client encounters this many connection errors, it exits and - # restarts. Retries are one minute apart. - connection_error_limit = 60 * 24 - - def __init__(self, heart_beat_cb=None): - self.heart_beat_cb = heart_beat_cb - self.proxies = self._GetProxies() - self.base_urls = self._GetBaseURLs() - - # We start checking with this proxy. - self.last_proxy_index = 0 - self.last_base_url_index = 0 - - # If we have connected previously but now suddenly fail to connect, we try - # the connection a few times (retry_error_limit) before we determine - # that it is failed. - self.consecutive_connection_errors = 0 - - self.active_base_url = None - self.error_poll_min = config.CONFIG["Client.error_poll_min"] - - def _GetBaseURLs(self): - """Gathers a list of base URLs we will try.""" - result = config.CONFIG["Client.server_urls"] - if not result: - # Backwards compatibility - deduce server_urls from Client.control_urls. - for control_url in config.CONFIG["Client.control_urls"]: - result.append(posixpath.dirname(control_url) + "/") - - # Check the URLs for trailing /. This traps configuration errors. - for url in result: - if not url.endswith("/"): - raise RuntimeError("Bad URL: %s URLs must end with /" % url) - - return result - - def _GetProxies(self): - """Gather a list of proxies to use.""" - # Detect proxies from the OS environment. - result = client_utils.FindProxies() - - # Also try to connect directly if all proxies fail. - result.append("") - - # Also try all proxies configured in the config system. - result.extend(config.CONFIG["Client.proxy_servers"]) - - return result - - def _ConcatenateURL(self, base, url): - if not url.startswith("/"): - url = "/" + url - - if base.endswith("/"): - base = base[:-1] - - return base + url - - def OpenServerEndpoint(self, - path, - verify_cb=lambda x: True, - data=None, - params=None, - headers=None, - method="GET", - timeout=None): - """Search through all the base URLs to connect to one that works. - - This is a thin wrapper around requests.request() so most parameters are - documented there. - - Args: - path: The URL path to access in this endpoint. - verify_cb: A callback which should return True if the response is - reasonable. This is used to detect if we are able to talk to the correct - endpoint. If not we try a different endpoint/proxy combination. - data: Parameters to send in POST bodies (See Requests documentation). - params: Parameters to send in GET URLs (See Requests documentation). - headers: Additional headers (See Requests documentation) - method: The HTTP method to use. If not set we select one automatically. - timeout: See Requests documentation. - - Returns: - an HTTPObject() instance with the correct error code set. - """ - tries = 0 - last_error = HTTPObject(code=404) - - while tries < len(self.base_urls): - base_url_index = self.last_base_url_index % len(self.base_urls) - active_base_url = self.base_urls[base_url_index] - - result = self.OpenURL( - self._ConcatenateURL(active_base_url, path), - data=data, - params=params, - headers=headers, - method=method, - timeout=timeout, - verify_cb=verify_cb, - ) - - if not result.Success(): - tries += 1 - self.last_base_url_index += 1 - last_error = result - continue - - # The URL worked - we record that. - self.active_base_url = active_base_url - - return result - - # No connection is possible at all. - logging.info( - "Could not connect to GRR servers %s, directly or through " - "these proxies: %s.", self.base_urls, self.proxies) - - return last_error - - def OpenURL(self, - url, - verify_cb=lambda x: True, - data=None, - params=None, - headers=None, - method="GET", - timeout=None): - """Get the requested URL. - - Note that we do not have any concept of timing here - we try to connect - through all proxies as fast as possible until one works. Timing and poll - frequency is left to the calling code. - - Args: - url: The URL to fetch - verify_cb: An optional callback which can be used to validate the URL. It - receives the HTTPObject and return True if this seems OK, False - otherwise. For example, if we are behind a captive portal we might - receive invalid object even though HTTP status is 200. - data: Parameters to send in POST bodies (See Requests documentation). - params: Parameters to send in GET URLs (See Requests documentation). - headers: Additional headers (See Requests documentation) - method: The HTTP method to use. If not set we select one automatically. - timeout: See Requests documentation. - - Returns: - An HTTPObject instance or None if a connection could not be made. - - """ - # Start checking the proxy from the last value found. - tries = 0 - last_error = 500 - - while tries < len(self.proxies): - proxy_index = self.last_proxy_index % len(self.proxies) - proxy = self.proxies[proxy_index] - try: - proxydict = {} - if proxy: - proxydict["http"] = proxy - proxydict["https"] = proxy - - headers = (headers or {}).copy() - headers["Cache-Control"] = "no-cache" - - if data: - method = "POST" - - duration, handle = self._RetryRequest( - url=url, - data=data, - params=params, - headers=headers, - method=method, - timeout=timeout, - proxies=proxydict, - ) - - data = handle.content - - result = HTTPObject( - url=url, data=data, proxy=proxy, code=200, duration=duration) - - if not verify_cb(result): - raise IOError("Data not verified.") - - # The last connection worked. - self.consecutive_connection_errors = 0 - return result - - except requests.RequestException as e: - # Especially trap a 406 error message - it means the client needs to - # enroll. - if e.response is not None: - last_error = e.response.status_code - if last_error == 406: - # A 406 is not considered an error as the frontend is reachable. If - # we considered it as an error the client would be unable to send - # the enrollment request since connection errors disable message - # draining. - self.consecutive_connection_errors = 0 - return HTTPObject(code=406) - - # Try the next proxy - self.last_proxy_index = proxy_index + 1 - tries += 1 - - # Catch any exceptions that dont have a code (e.g. socket.error). - except IOError: - # Try the next proxy - self.last_proxy_index = proxy_index + 1 - tries += 1 - last_error = 500 - # Catch unexpected exceptions. If the error is proxy related it makes - # sense to cycle the proxy before reraising. One error we have seen here - # is ProxySchemeUnknown but urllib can raise many different exceptions, it - # doesn't make sense to enumerate them all. - except Exception: # pylint: disable=broad-except - logging.exception( - "Got an unexpected exception while connecting to the server.") - # Try the next proxy - self.last_proxy_index = proxy_index + 1 - tries += 1 - last_error = 500 - - # We failed to connect at all here. - return HTTPObject(code=last_error) - - def _RetryRequest(self, timeout=None, **request_args): - """Retry the request a few times before we determine it failed. - - Sometimes the frontend becomes loaded and issues a 500 error to throttle the - clients. We wait Client.error_poll_min seconds between each attempt to back - off the frontend. Note that this does not affect any timing algorithm in the - client itself which is controlled by the Timer() class. - - Args: - timeout: Timeout for retry. - **request_args: Args to the requests.request call. - - Returns: - a tuple of duration, urllib.request.urlopen response. - """ - while True: - try: - now = time.time() - if not timeout: - timeout = config.CONFIG["Client.http_timeout"] - - result = requests.request(**request_args) - # By default requests doesn't raise on HTTP error codes. - result.raise_for_status() - - # Requests does not always raise an exception when an incorrect response - # is received. This fixes that behaviour. - if not result.ok: - raise requests.RequestException(response=result) - - return time.time() - now, result - - # Catch any exceptions that dont have a code (e.g. socket.error). - except IOError as e: - self.consecutive_connection_errors += 1 - # Request failed. If we connected successfully before we attempt a few - # connections before we determine that it really failed. This might - # happen if the front end is loaded and returns a few throttling 500 - # messages. - if self.active_base_url is not None: - # Propagate 406 immediately without retrying, as 406 is a valid - # response that indicates a need for enrollment. - response = getattr(e, "response", None) - if getattr(response, "status_code", None) == 406: - raise - - if self.consecutive_connection_errors >= self.retry_error_limit: - # We tried several times but this really did not work, just fail it. - logging.info( - "Too many connection errors to %s, retrying another URL", - self.active_base_url) - self.active_base_url = None - raise e - - # Back off hard to allow the front end to recover. - logging.debug( - "Unable to connect to frontend. Backing off %s seconds.", - self.error_poll_min) - self.Wait(self.error_poll_min) - - # We never previously connected, maybe the URL/proxy is wrong? Just fail - # right away to allow callers to try a different URL. - else: - raise e - - def Wait(self, timeout): - """Wait for the specified timeout.""" - time.sleep(timeout - int(timeout)) - - if self.heart_beat_cb: - self.heart_beat_cb() - - # Split a long sleep interval into 1 second intervals so we can heartbeat. - for _ in range(int(timeout)): - time.sleep(1) - - if self.heart_beat_cb: - self.heart_beat_cb() - - def ErrorLimitReached(self): - return self.consecutive_connection_errors > self.connection_error_limit - - -class Timer(object): - """Implements the polling policy. - - External code simply calls our Wait() method without regard to the exact - timing policy. - """ - - # Slew of poll time. - poll_slew = 1.15 - - def __init__(self): - self.poll_min = config.CONFIG["Client.poll_min"] - self.sleep_time = self.poll_max = config.CONFIG["Client.poll_max"] - - def FastPoll(self): - """Switch to fast poll mode.""" - self.sleep_time = self.poll_min - - def SlowPoll(self): - """Switch to slow poll mode.""" - self.sleep_time = self.poll_max - - def Wait(self): - """Wait until the next action is needed.""" - time.sleep(self.sleep_time - int(self.sleep_time)) - - # Split a long sleep interval into 1 second intervals so we can heartbeat. - for _ in range(int(self.sleep_time)): - time.sleep(1) - - # Back off slowly at first and fast if no answer. - self.sleep_time = min(self.poll_max, - max(self.poll_min, self.sleep_time) * self.poll_slew) - - class GRRClientWorker(threading.Thread): """This client worker runs the main loop in another thread. @@ -520,11 +69,6 @@ def HeartBeatStub(): self.lock = threading.RLock() - # The worker may communicate over HTTP independently from the comms - # thread. This way we do not need to synchronize the HTTP manager between - # the two threads. - self.http_manager = HTTPManager(heart_beat_cb=heart_beat_cb) - # This queue should never hit its maximum since the server will throttle # messages before this. self._in_queue = utils.HeartbeatQueue(callback=heart_beat_cb, maxsize=1024) @@ -878,486 +422,3 @@ def Size(self): def Full(self): return self._total_size >= self._maxsize - - -class GRRHTTPClient(object): - """A class which abstracts away HTTP communications. - - To create a new GRR HTTP client, instantiate this class and generate - its Run() method. - - The HTTP client starts up by loading a communicator which will read the - client's public key (or create a new random key). Since the client ID is based - on the key (it's a hash of the public key), the communicator controls the - client name. - - The client worker is then created - this will be the main thread for executing - server messages. - - The client then creates a HTTPManager() instance to control communication with - the front end over HTTP, and a Timer() instance to control polling policy. - - The HTTP client simply reads pending messages from the client worker queues - and makes POST requests to the server. The POST request may return the - following error conditions: - - - A successful POST is signified by a status of 200: The client worker is - given any requests the server has sent. - - - A status code of 406 means that the server is unable to communicate with - the client. The client will then prepare an enrollment request CSR and - send that. Enrollment requests are throttled to a - maximum of one every 10 minutes. - - - A status code of 500 is an error, the messages are re-queued and the - client waits and retries to send them later. - """ - - http_manager_class = HTTPManager - - def __init__(self, ca_cert=None, worker_cls=None, private_key=None): - """Constructor. - - Args: - ca_cert: String representation of a CA certificate to use for checking - server certificate. - worker_cls: The client worker class to use. Defaults to GRRClientWorker. - private_key: The private key for this client. Defaults to config - Client.private_key. - """ - self.ca_cert = ca_cert - if private_key is None: - private_key = config.CONFIG.Get("Client.private_key", default=None) - - # The server's PEM encoded certificate. - self.server_certificate = None - - # This manages our HTTP connections. Note: The comms thread is allowed to - # block indefinitely since the worker thread is responsible for - # heart-beating the nanny. We assume that HTTP requests can not block - # indefinitely. - self.http_manager = self.http_manager_class() - - # The communicator manages our crypto with the server. - self.communicator = ClientCommunicator(private_key=private_key) - - # This controls our polling frequency. - self.timer = Timer() - - # The time we last sent an enrollment request. Enrollment requests are - # throttled especially to a maximum of one every 10 minutes. - self.last_enrollment_time = 0 - - # The time we last checked with the foreman. - self.last_foreman_check = 0 - - # The client worker does all the real work here. - if worker_cls: - self.client_worker = worker_cls(client=self) - else: - self.client_worker = GRRClientWorker(client=self) - # TODO(hanuszczak): Maybe we should start the thread in `GRRHTTPClient::Run` - # method instead? Starting threads in constructor is rarely a good idea, is - # it guaranteed that we call `GRRHTTPClient::Run` only once? - self.client_worker.start() - - def FleetspeakEnabled(self): - return False - - def VerifyServerPEM(self, http_object): - """Check the server PEM for validity. - - This is used to determine connectivity to the server. Sometimes captive - portals return a valid HTTP status, but the data is corrupted. - - Args: - http_object: The response received from the server. - - Returns: - True if the response contains a valid server certificate. - """ - try: - server_pem = http_object.data - server_url = http_object.url - - if b"BEGIN CERTIFICATE" in server_pem: - # Now we know that this proxy is working. We still have to verify the - # certificate. This will raise if the server cert is invalid. - server_certificate = rdf_crypto.RDFX509Cert(server_pem) - self.communicator.LoadServerCertificate( - server_certificate=server_certificate, ca_certificate=self.ca_cert) - - logging.info("Server PEM re-keyed.") - return True - except Exception as e: # pylint: disable=broad-except - logging.info("Unable to verify server certificate at %s: %s", server_url, - e) - - return False - - def VerifyServerControlResponse(self, http_object): - """Verify the server response to a 'control' endpoint POST message. - - We consider the message correct if and only if we can decrypt it - properly. Note that in practice we can not use the HTTP status to figure out - if the request worked because captive proxies have a habit of lying and - returning a HTTP success code even when there is no connectivity. - - Args: - http_object: The HTTPObject returned from the HTTP transaction. - - Returns: - True if the http_object is correct. False if it is not valid. - - Side Effect: - Fill in the decoded_data attribute in the http_object. - """ - if http_object.code != 200: - return False - - # Try to decrypt the message into the http_object. - try: - http_object.messages, http_object.source, http_object.nonce = ( - self.communicator.DecryptMessage(http_object.data)) - - return True - - # Something went wrong - the response seems invalid! - except communicator.DecodingError as e: - logging.info("Protobuf decode error: %s.", e) - return False - - def MakeRequest(self, data): - """Make a HTTP Post request to the server 'control' endpoint.""" - communicator.GRR_CLIENT_SENT_BYTES.Increment(len(data)) - - # Verify the response is as it should be from the control endpoint. - response = self.http_manager.OpenServerEndpoint( - path="control?api=%s" % config.CONFIG["Network.api"], - verify_cb=self.VerifyServerControlResponse, - data=data, - headers={"Content-Type": "binary/octet-stream"}) - - if response.code == 406: - self.InitiateEnrolment() - return response - - if response.code == 200: - communicator.GRR_CLIENT_RECEIVED_BYTES.Increment(len(response.data)) - return response - - # An unspecified error occurred. - return response - - def RunOnce(self): - """Makes a single request to the GRR server. - - Returns: - A Status() object indicating how the last POST went. - """ - # Attempt to fetch and load server certificate. - if not self._FetchServerCertificate(): - self.timer.Wait() - return HTTPObject(code=500) - - # Here we only drain messages if we were able to connect to the server in - # the last poll request. Otherwise we just wait until the connection comes - # back so we don't expire our messages too fast. - if self.http_manager.consecutive_connection_errors == 0: - # Grab some messages to send - message_list = self.client_worker.Drain( - max_size=config.CONFIG["Client.max_post_size"]) - else: - message_list = rdf_flows.MessageList() - - # If any outbound messages require fast poll we switch to fast poll mode. - for message in message_list.job: - if message.require_fastpoll: - self.timer.FastPoll() - break - - # Make new encrypted ClientCommunication rdfvalue. - payload = rdf_flows.ClientCommunication() - - # If our memory footprint is too large, we advertise that our input queue - # is full. This will prevent the server from sending us any messages, and - # hopefully allow us to work down our memory usage, by processing any - # outstanding messages. - if self.client_worker.MemoryExceeded(): - logging.info("Memory exceeded, will not retrieve jobs.") - payload.queue_size = 1000000 - else: - # Let the server know how many messages are currently queued in - # the input queue. - payload.queue_size = self.client_worker.InQueueSize() - - nonce = self.communicator.EncodeMessages(message_list, payload) - payload_data = payload.SerializeToBytes() - response = self.MakeRequest(payload_data) - - # Unable to decode response or response not valid. - if response.code != 200 or response.messages is None: - # We don't print response here since it should be encrypted and will - # cause ascii conversion errors. - logging.info("%s: Could not connect to server at %s, status %s", - self.communicator.common_name, - self.http_manager.active_base_url, response.code) - - # Force the server pem to be reparsed on the next connection. - self.server_certificate = None - - # Reschedule the tasks back on the queue so they get retried next time. - messages = list(message_list.job) - for message in messages: - message.require_fastpoll = False - message.ttl -= 1 - if message.ttl > 0: - self.client_worker.QueueResponse(message) - else: - logging.info("Dropped message due to retransmissions.") - - return response - - # Check the decoded nonce was as expected. - if response.nonce != nonce: - logging.info("Nonce not matched.") - response.code = 500 - return response - - if response.source != self.communicator.server_name: - logging.info("Received a message not from the server " - "%s, expected %s.", response.source, - self.communicator.server_name) - response.code = 500 - return response - - # Check to see if any inbound messages want us to fastpoll. This means we - # drop to fastpoll immediately on a new request rather than waiting for the - # next beacon to report results. - for message in response.messages: - if message.require_fastpoll: - self.timer.FastPoll() - break - - # Process all messages. Messages can be processed by clients in - # any order since clients do not have state. - self.client_worker.QueueMessages(response.messages) - - cn = self.communicator.common_name - logging.info( - "%s: Sending %s(%s), Received %s messages in %s sec. " - "Sleeping for %s sec.", cn, len(message_list), len(payload_data), - len(response.messages), response.duration, self.timer.sleep_time) - - return response - - def SendForemanRequest(self): - self.client_worker.SendReply( - rdf_protodict.DataBlob(), - session_id=rdfvalue.FlowSessionID(flow_name="Foreman"), - require_fastpoll=False) - - def _FetchServerCertificate(self): - """Attempts to fetch the server cert. - - Returns: - True if we succeed. - """ - # Certificate is loaded and still valid. - if self.server_certificate: - return True - - response = self.http_manager.OpenServerEndpoint( - "server.pem", verify_cb=self.VerifyServerPEM) - - if response.Success(): - self.server_certificate = response.data - return True - - # We failed to fetch the cert, switch to slow poll mode. - self.timer.SlowPoll() - return False - - def Run(self): - """The main run method of the client. - - This method does not normally return. Only if there have been more than - connection_error_limit failures, the method returns and allows the - client to exit. - """ - while True: - if self.http_manager.ErrorLimitReached(): - return - - now = time.time() - # Check with the foreman if we need to - if (now > self.last_foreman_check + - config.CONFIG["Client.foreman_check_frequency"]): - # We must not queue messages from the comms thread with blocking=True - # or we might deadlock. If the output queue is full, we can't accept - # more work from the foreman anyways so it's ok to drop the message. - try: - self.client_worker.SendReply( - rdf_protodict.DataBlob(), - session_id=rdfvalue.FlowSessionID(flow_name="Foreman"), - require_fastpoll=False, - blocking=False) - self.last_foreman_check = now - except queue.Full: - pass - - try: - self.RunOnce() - except Exception: # pylint: disable=broad-except - # Catch everything, yes, this is terrible but necessary - logging.warning("Uncaught exception caught: %s", traceback.format_exc()) - if flags.FLAGS.pdb_post_mortem: - pdb.post_mortem() - - # We suicide if our memory is exceeded, and there is no more work to do - # right now. Our death should not result in loss of messages since we are - # not holding any requests in our input queues. - if (self.client_worker.MemoryExceeded() and - not self.client_worker.IsActive() and - self.client_worker.InQueueSize() == 0 and - self.client_worker.OutQueueSize() == 0): - logging.warning("Memory exceeded - exiting.") - self.client_worker.SendClientAlert("Memory limit exceeded, exiting.") - # Make sure this will return True so we don't get more work. - # pylint: disable=g-bad-name - self.client_worker.MemoryExceeded = lambda: True - # pylint: enable=g-bad-name - # Now send back the client message. - self.RunOnce() - # And done for now. - sys.exit(-1) - - self.timer.Wait() - self.client_worker.Heartbeat() - - def InitiateEnrolment(self): - """Initiate the enrollment process. - - We do not sent more than one enrollment request every 10 minutes. Note that - we still communicate to the server in fast poll mode, but these requests are - not carrying any payload. - """ - logging.debug("sending enrollment request") - now = time.time() - if now > self.last_enrollment_time + 10 * 60: - if not self.last_enrollment_time: - # This is the first enrollment request - we should enter fastpoll mode. - self.timer.FastPoll() - - self.last_enrollment_time = now - # Send registration request: - self.client_worker.SendReply( - rdf_crypto.Certificate( - type=rdf_crypto.Certificate.Type.CSR, - pem=self.communicator.GetCSRAsPem()), - session_id=rdfvalue.SessionID( - queue=queues.ENROLLMENT, flow_name="Enrol")) - - -class ClientCommunicator(communicator.Communicator): - """A communicator implementation for clients. - - This extends the generic communicator to include verification of - server side certificates. - """ - - def __init__(self, certificate=None, private_key=None): - super().__init__(certificate=certificate, private_key=private_key) - self.InitPrivateKey() - - def InitPrivateKey(self): - """Makes sure this client has a private key set. - - It first tries to load an RSA key from the certificate. - - If no certificate is found, or it is invalid, we make a new random RSA key, - and store it as our certificate. - - Returns: - An RSA key - either from the certificate or a new random key. - """ - if self.private_key: - try: - self.common_name = rdf_client.ClientURN.FromPrivateKey(self.private_key) - - logging.info("Starting client %s", self.common_name) - - return self.private_key - - except type_info.TypeValueError: - pass - - # We either have an invalid key or no key. We just generate a new one. - key = rdf_crypto.RSAPrivateKey.GenerateKey( - bits=config.CONFIG["Client.rsa_key_length"]) - - self.common_name = rdf_client.ClientURN.FromPrivateKey(key) - logging.info("Client pending enrolment %s", self.common_name) - - # Save the keys - self.SavePrivateKey(key) - - return key - - def GetCSR(self): - """Return our CSR.""" - return rdf_crypto.CertificateSigningRequest( - common_name=self.common_name, private_key=self.private_key) - - def GetCSRAsPem(self): - """Return our CSR in PEM format.""" - return self.GetCSR().AsPEM() - - def SavePrivateKey(self, private_key): - """Store the new private key on disk.""" - self.private_key = private_key - config.CONFIG.Set("Client.private_key", - self.private_key.AsPEM().decode("ascii")) - config.CONFIG.Write() - - def LoadServerCertificate(self, server_certificate=None, ca_certificate=None): - """Loads and verifies the server certificate.""" - # Check that the server certificate verifies - try: - server_certificate.Verify(ca_certificate.GetPublicKey()) - except rdf_crypto.VerificationError as e: - self.server_name = None - raise IOError("Server cert is invalid: %s" % e) - - # Make sure that the serial number is higher. - server_cert_serial = server_certificate.GetSerialNumber() - - if server_cert_serial < config.CONFIG["Client.server_serial_number"]: - # We can not accept this serial number... - raise IOError("Server certificate serial number is too old.") - elif server_cert_serial > config.CONFIG["Client.server_serial_number"]: - logging.info("Server serial number updated to %s", server_cert_serial) - config.CONFIG.Set("Client.server_serial_number", server_cert_serial) - - # Save the new data to the config file. - config.CONFIG.Write() - - self.server_name = server_certificate.GetCN() - self.server_certificate = server_certificate - self.ca_certificate = ca_certificate - self.server_public_key = server_certificate.GetPublicKey() - # If we still have a cached session key, we need to remove it. - self._ClearServerCipherCache() - - def EncodeMessages(self, message_list, result, **kwargs): - # Force the right API to be used - kwargs["api_version"] = config.CONFIG["Network.api"] - return super().EncodeMessages(message_list, result, **kwargs) - - def _GetRemotePublicKey(self, common_name): - - if common_name == self.server_name: - return self.server_public_key - - raise communicator.UnknownServerCertError( - "Client wants to talk to %s, not %s" % (common_name, self.server_name)) diff --git a/grr/client/grr_response_client/comms_test.py b/grr/client/grr_response_client/comms_test.py index ddf2768ae7..4752da9242 100644 --- a/grr/client/grr_response_client/comms_test.py +++ b/grr/client/grr_response_client/comms_test.py @@ -1,348 +1,15 @@ #!/usr/bin/env python """Test for client comms.""" -import queue -import time from unittest import mock from absl import app -import requests from grr_response_client import comms from grr_response_core.lib import rdfvalue -from grr_response_core.lib import utils -from grr_response_core.lib.rdfvalues import flows as rdf_flows from grr.test_lib import test_lib -def _make_http_response(code=200): - """A helper for creating HTTP responses.""" - response = requests.Response() - response.status_code = code - return response - - -def _make_404(): - return _make_http_response(404) - - -def _make_200(content): - response = _make_http_response(200) - response._content = content - return response - - -class RequestsInstrumentor(object): - """Instrument the `requests` library.""" - - def __init__(self): - self.time = 0 - self.current_opener = None - # Record the actions in order. - self.actions = [] - - # These are the responses we will do. - self.responses = [] - - def request(self, **request_options): - self.actions.append([self.time, request_options]) - if self.responses: - response = self.responses.pop(0) - if isinstance(response, IOError): - raise response - return response - else: - return _make_404() - - def sleep(self, timeout): - self.time += timeout - - def instrument(self): - """Install the mocks required. - - Returns: - A context manager that when exits restores the mocks. - """ - self.actions = [] - return utils.MultiStubber((requests, "request", self.request), - (time, "sleep", self.sleep)) - - -class URLFilter(RequestsInstrumentor): - """Emulate only a single server url that works.""" - - def request(self, url=None, **kwargs): - # If request is from server2 - return a valid response. Assume, server2 is - # reachable from all proxies. - response = super().request(url=url, **kwargs) - if "server2" in url: - return _make_200("Good") - return response - - -class MockHTTPManager(comms.HTTPManager): - - def _GetBaseURLs(self): - return ["http://server1/", "http://server2/", "http://server3/"] - - def _GetProxies(self): - """Do not test the proxy gathering logic itself.""" - return ["proxy1", "proxy2", "proxy3"] - - -class HTTPManagerTest(test_lib.GRRBaseTest): - """Tests the HTTP Manager.""" - - def MakeRequest(self, instrumentor, manager, path, verify_cb=lambda x: True): - with utils.MultiStubber((requests, "request", instrumentor.request), - (time, "sleep", instrumentor.sleep)): - return manager.OpenServerEndpoint(path, verify_cb=verify_cb) - - def testBaseURLConcatenation(self): - instrumentor = RequestsInstrumentor() - with instrumentor.instrument(): - manager = MockHTTPManager() - manager.OpenServerEndpoint("/control") - - # Make sure that the URL is concatenated properly (no //). - self.assertEqual(instrumentor.actions[0][1]["url"], - "http://server1/control") - - def testProxySearch(self): - """Check that all proxies will be searched in order.""" - # Do not specify a response - all requests will return a 404 message. - instrumentor = RequestsInstrumentor() - with instrumentor.instrument(): - manager = MockHTTPManager() - result = manager.OpenURL("http://www.google.com/") - - # Three requests are made. - proxies = [x[1]["proxies"]["https"] for x in instrumentor.actions] - self.assertEqual(proxies, manager.proxies) - - # Result is an error since no requests succeeded. - self.assertEqual(result.code, 404) - - def testVerifyCB(self): - """Check that we can handle captive portals via the verify CB. - - Captive portals do not cause an exception but return bad data. - """ - - def verify_cb(http_object): - return http_object.data == "Good" - - instrumentor = RequestsInstrumentor() - - # First request is an exception, next is bad and the last is good. - instrumentor.responses = [_make_404(), _make_200("Bad"), _make_200("Good")] - with instrumentor.instrument(): - manager = MockHTTPManager() - result = manager.OpenURL("http://www.google.com/", verify_cb=verify_cb) - - self.assertEqual(result.data, "Good") - - def testURLSwitching(self): - """Ensure that the manager switches URLs to one that works.""" - # Only server2 works and returns Good response. - instrumentor = URLFilter() - with instrumentor.instrument(): - manager = MockHTTPManager() - result = manager.OpenServerEndpoint("control") - - # The result is correct. - self.assertEqual(result.data, "Good") - - queries = [ - (x[1]["url"], x[1]["proxies"]["http"]) for x in instrumentor.actions - ] - - self.assertEqual( - queries, - # First search for server1 through all proxies. - [ - ("http://server1/control", "proxy1"), - ("http://server1/control", "proxy2"), - ("http://server1/control", "proxy3"), - - # Now search for server2 through all proxies. - ("http://server2/control", "proxy1") - ]) - - def testTemporaryFailure(self): - """If the front end gives an intermittent 500, we must back off.""" - instrumentor = RequestsInstrumentor() - # First response good, then a 500 error, then another good response. - instrumentor.responses = [ - _make_200("Good"), - _make_http_response(code=500), - _make_200("Also Good") - ] - - manager = MockHTTPManager() - with instrumentor.instrument(): - # First request - should be fine. - result = manager.OpenServerEndpoint("control") - - self.assertEqual(result.data, "Good") - - with instrumentor.instrument(): - # Second request - should appear fine. - result = manager.OpenServerEndpoint("control") - - self.assertEqual(result.data, "Also Good") - - # But we actually made two requests. - self.assertLen(instrumentor.actions, 2) - - # And we waited 60 seconds to make the second one. - self.assertEqual(instrumentor.actions[0][0], 0) - self.assertEqual(instrumentor.actions[1][0], manager.error_poll_min) - - # Make sure that the manager cleared its consecutive_connection_errors. - self.assertEqual(manager.consecutive_connection_errors, 0) - - def test406Errors(self): - """Ensure that 406 enrollment requests are propagated immediately. - - Enrollment responses (406) are sent by the server when the client is not - suitable enrolled. The http manager should treat those as correct responses - and stop searching for proxy/url combinations in order to allow the client - to commence enrollment workflow. - """ - instrumentor = RequestsInstrumentor() - instrumentor.responses = [_make_http_response(code=406)] - - manager = MockHTTPManager() - with instrumentor.instrument(): - # First request - should raise a 406 error. - result = manager.OpenServerEndpoint("control") - - self.assertEqual(result.code, 406) - - # We should not search for proxy/url combinations. - self.assertLen(instrumentor.actions, 1) - - # A 406 message is not considered an error. - self.assertEqual(manager.consecutive_connection_errors, 0) - - def testConnectionErrorRecovery(self): - instrumentor = RequestsInstrumentor() - - # When we can't connect at all (server not listening), we get a - # requests.exceptions.ConnectionError but the response object is None. - err_response = requests.ConnectionError("Error", response=None) - instrumentor.responses = [err_response, _make_200("Good")] - with instrumentor.instrument(): - manager = MockHTTPManager() - result = manager.OpenServerEndpoint("control") - - self.assertEqual(result.data, "Good") - - -class SizeLimitedQueueTest(test_lib.GRRBaseTest): - - def testSizeLimitedQueue(self): - - limited_queue = comms.SizeLimitedQueue( - maxsize=10000000, heart_beat_cb=lambda: None) - - msg_a = rdf_flows.GrrMessage(name="A") - msg_b = rdf_flows.GrrMessage(name="B") - msg_c = rdf_flows.GrrMessage(name="C") - - for _ in range(10): - limited_queue.Put(msg_a) - limited_queue.Put(msg_b) - limited_queue.Put(msg_c) - - result = limited_queue.GetMessages() - self.assertCountEqual(list(result.job), [msg_c] * 10 + [msg_a, msg_b] * 10) - - # Tests a partial Get(). - for _ in range(7): - limited_queue.Put(msg_a) - limited_queue.Put(msg_b) - limited_queue.Put(msg_c) - - result = limited_queue.GetMessages( - soft_size_limit=len(msg_a.SerializeToBytes()) * 5 - 1) - - self.assertLen(list(result.job), 5) - - for _ in range(3): - limited_queue.Put(msg_a) - limited_queue.Put(msg_b) - limited_queue.Put(msg_c) - - # Append the remaining messages to the same result. - result.job.Extend(limited_queue.GetMessages().job) - self.assertCountEqual(list(result.job), [msg_c] * 10 + [msg_a, msg_b] * 10) - - def testSizeLimitedQueueSize(self): - - q = comms.SizeLimitedQueue(1000) - msg_a = rdf_flows.GrrMessage(name="A") - msg_b = rdf_flows.GrrMessage(name="B") - msg_c = rdf_flows.GrrMessage(name="C") - msg_d = rdf_flows.GrrMessage(name="D") - messages = [msg_a, msg_b, msg_c, msg_d] - - in_queue = [] - self.assertEqual(q.Size(), 0) - - for m in messages: - q.Put(m, block=False) - in_queue.append(m) - - self.assertEqual(q.Size(), - sum([len(m.SerializeToBytes()) for m in in_queue])) - - for _ in range(len(messages)): - msg_list = q.GetMessages(1) - self.assertLen(msg_list.job, 1) - in_queue.remove(msg_list.job[0]) - - self.assertEqual(q.Size(), - sum([len(m.SerializeToBytes()) for m in in_queue])) - - def testSizeLimitedQueueOverflow(self): - - msg_a = rdf_flows.GrrMessage(name="A") - msg_b = rdf_flows.GrrMessage(name="B") - msg_c = rdf_flows.GrrMessage(name="C") - msg_d = rdf_flows.GrrMessage(name="D") - - limited_queue = comms.SizeLimitedQueue( - maxsize=3 * len(msg_a.SerializeToBytes()), heart_beat_cb=lambda: None) - - limited_queue.Put(msg_a, block=False) - limited_queue.Put(msg_b, block=False) - limited_queue.Put(msg_c, block=False) - with self.assertRaises(queue.Full): - limited_queue.Put(msg_d, block=False) - - def testSizeLimitedQueueHeartbeat(self): - - msg_a = rdf_flows.GrrMessage(name="A") - msg_b = rdf_flows.GrrMessage(name="B") - msg_c = rdf_flows.GrrMessage(name="C") - msg_d = rdf_flows.GrrMessage(name="D") - - heartbeat = mock.Mock() - - limited_queue = comms.SizeLimitedQueue( - maxsize=3 * len(msg_a.SerializeToBytes()), heart_beat_cb=heartbeat) - - limited_queue.Put(msg_a) - limited_queue.Put(msg_b) - limited_queue.Put(msg_c) - with self.assertRaises(queue.Full): - limited_queue.Put(msg_d, timeout=1) - - self.assertTrue(heartbeat.called) - - class GRRClientWorkerTest(test_lib.GRRBaseTest): """Tests the GRRClientWorker class.""" diff --git a/grr/client/grr_response_client/communicator.py b/grr/client/grr_response_client/communicator.py deleted file mode 100644 index 57478a841f..0000000000 --- a/grr/client/grr_response_client/communicator.py +++ /dev/null @@ -1,294 +0,0 @@ -#!/usr/bin/env python -"""Abstracts encryption and authentication.""" - -import abc -import struct -import time -import zlib - -from grr_response_core.lib import communicator -from grr_response_core.lib import rdfvalue -from grr_response_core.lib import type_info -from grr_response_core.lib.rdfvalues import flows as rdf_flows -from grr_response_core.stats import metrics - - -Error = communicator.Error -DecodingError = communicator.Error -DecryptionError = communicator.DecryptionError -LegacyClientDecryptionError = communicator.LegacyClientDecryptionError - -GRR_CLIENT_RECEIVED_BYTES = metrics.Counter("grr_client_received_bytes") -GRR_CLIENT_SENT_BYTES = metrics.Counter("grr_client_sent_bytes") - - -class UnknownServerCertError(DecodingError): - """Raised when the client key is not retrieved.""" - - -class Communicator(metaclass=abc.ABCMeta): - """A class responsible for encoding and decoding comms.""" - server_name = None - common_name = None - - def __init__(self, certificate=None, private_key=None): - """Creates a communicator. - - Args: - certificate: Our own certificate. - private_key: Our own private key. - """ - self.private_key = private_key - self.certificate = certificate - self._ClearServerCipherCache() - - def _ClearServerCipherCache(self): - self.server_cipher = None - self.server_cipher_age = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(0) - - @abc.abstractmethod - def _GetRemotePublicKey(self, server_name): - raise NotImplementedError() - - @classmethod - def EncodeMessageList(cls, message_list, packed_message_list): - """Encode the MessageList into the packed_message_list rdfvalue.""" - # By default uncompress - uncompressed_data = message_list.SerializeToBytes() - packed_message_list.message_list = uncompressed_data - - compressed_data = zlib.compress(uncompressed_data) - - # Only compress if it buys us something. - if len(compressed_data) < len(uncompressed_data): - packed_message_list.compression = ( - rdf_flows.PackedMessageList.CompressionType.ZCOMPRESSION) - packed_message_list.message_list = compressed_data - - def _GetServerCipher(self): - """Returns the cipher for self.server_name.""" - - if self.server_cipher is not None: - expiry = self.server_cipher_age + rdfvalue.Duration.From(1, rdfvalue.DAYS) - if expiry > rdfvalue.RDFDatetime.Now(): - return self.server_cipher - - remote_public_key = self._GetRemotePublicKey(self.server_name) - self.server_cipher = communicator.Cipher(self.common_name, self.private_key, - remote_public_key) - self.server_cipher_age = rdfvalue.RDFDatetime.Now() - return self.server_cipher - - def EncodeMessages(self, - message_list, - result, - timestamp=None, - api_version=3): - """Accepts a list of messages and encodes for transmission. - - This function signs and then encrypts the payload. - - Args: - message_list: A MessageList rdfvalue containing a list of GrrMessages. - result: A ClientCommunication rdfvalue which will be filled in. - timestamp: A timestamp to use for the signed messages. If None - use the - current time. - api_version: The api version which this should be encoded in. - - Returns: - A nonce (based on time) which is inserted to the encrypted payload. The - client can verify that the server is able to decrypt the message and - return the nonce. - - Raises: - RuntimeError: If we do not support this api version. - """ - if api_version not in [3]: - raise RuntimeError( - "Unsupported api version: %s, expected 3." % api_version) - cipher = self._GetServerCipher() - - # Make a nonce for this transaction - if timestamp is None: - self.timestamp = timestamp = int(time.time() * 1000000) - - packed_message_list = rdf_flows.PackedMessageList(timestamp=timestamp) - self.EncodeMessageList(message_list, packed_message_list) - - result.encrypted_cipher_metadata = cipher.encrypted_cipher_metadata - - # Include the encrypted cipher. - result.encrypted_cipher = cipher.encrypted_cipher - - serialized_message_list = packed_message_list.SerializeToBytes() - - # Encrypt the message symmetrically. - # New scheme cipher is signed plus hmac over message list. - result.packet_iv, result.encrypted = cipher.Encrypt(serialized_message_list) - - # This is to support older endpoints. - result.hmac = cipher.HMAC(result.encrypted) - - # Newer endpoints only look at this HMAC. It is recalculated for each packet - # in the session. Note that encrypted_cipher and encrypted_cipher_metadata - # do not change between all packets in this session. - result.full_hmac = cipher.HMAC(result.encrypted, result.encrypted_cipher, - result.encrypted_cipher_metadata, - result.packet_iv.SerializeToBytes(), - struct.pack(" None: + """Encode the MessageList into the packed_message_list rdfvalue.""" + # By default uncompress + uncompressed_data = message_list.SerializeToBytes() + packed_message_list.message_list = uncompressed_data + + compressed_data = zlib.compress(uncompressed_data) + + # Only compress if it buys us something. + if len(compressed_data) < len(uncompressed_data): + packed_message_list.compression = ( + rdf_flows.PackedMessageList.CompressionType.ZCOMPRESSION + ) + packed_message_list.message_list = compressed_data + + class GRRFleetspeakClient(object): """A Fleetspeak enabled client implementation.""" @@ -105,9 +126,6 @@ def _RunInLoop(self, loop_op): # This will terminate execution in the current thread. raise e - def FleetspeakEnabled(self): - return True - def Run(self): """The main run method of the client.""" for thread in self._threads.values(): @@ -135,8 +153,7 @@ def _ForemanOp(self): def _SendMessages(self, grr_msgs, background=False): """Sends a block of messages through Fleetspeak.""" message_list = rdf_flows.PackedMessageList() - communicator.Communicator.EncodeMessageList( - rdf_flows.MessageList(job=grr_msgs), message_list) + _EncodeMessageList(rdf_flows.MessageList(job=grr_msgs), message_list) fs_msg = fs_common_pb2.Message( message_type="MessageList", destination=fs_common_pb2.Address(service_name="GRR"), @@ -163,7 +180,7 @@ def _SendMessages(self, grr_msgs, background=False): logging.critical("Broken local Fleetspeak connection (write end).") raise - communicator.GRR_CLIENT_SENT_BYTES.Increment(sent_bytes) + client_metrics.GRR_CLIENT_SENT_BYTES.Increment(sent_bytes) def _SendOp(self): """Sends messages through Fleetspeak.""" @@ -209,7 +226,7 @@ def _ReceiveOp(self): "Unexpected proto type received through Fleetspeak: %r; expected " "grr.GrrMessage." % received_type) - communicator.GRR_CLIENT_RECEIVED_BYTES.Increment(received_bytes) + client_metrics.GRR_CLIENT_RECEIVED_BYTES.Increment(received_bytes) grr_msg = rdf_flows.GrrMessage.FromSerializedBytes(fs_msg.data.value) # Authentication is ensured by Fleetspeak. diff --git a/grr/client/grr_response_client/fleetspeak_client_test.py b/grr/client/grr_response_client/fleetspeak_client_test.py index 6a896ccd9c..d61e2a3c80 100644 --- a/grr/client/grr_response_client/fleetspeak_client_test.py +++ b/grr/client/grr_response_client/fleetspeak_client_test.py @@ -1,20 +1,44 @@ #!/usr/bin/env python import logging - from unittest import mock +import zlib from absl import app from absl.testing import absltest from grr_response_client import comms -from grr_response_client import communicator from grr_response_client import fleetspeak_client +from grr_response_core.lib import rdfvalue from grr_response_core.lib.rdfvalues import flows as rdf_flows from grr.test_lib import test_lib from fleetspeak.src.common.proto.fleetspeak import common_pb2 as fs_common_pb2 from fleetspeak.client_connector import connector as fs_client +def _DecompressMessageList( + packed_message_list: rdf_flows.PackedMessageList, +) -> rdf_flows.MessageList: + """Decompress the message data from packed_message_list.""" + compression = packed_message_list.compression + if compression == rdf_flows.PackedMessageList.CompressionType.UNCOMPRESSED: + data = packed_message_list.message_list + + elif compression == rdf_flows.PackedMessageList.CompressionType.ZCOMPRESSION: + try: + data = zlib.decompress(packed_message_list.message_list) + except zlib.error as e: + raise RuntimeError("Failed to decompress: %s" % e) from e + else: + raise RuntimeError("Compression scheme not supported") + + try: + result = rdf_flows.MessageList.FromSerializedBytes(data) + except rdfvalue.DecodeError as e: + raise RuntimeError("RDFValue parsing failed.") from e + + return result + + class FleetspeakClientTest(absltest.TestCase): @mock.patch.object(fs_client, "FleetspeakConnection") @@ -69,9 +93,11 @@ def testSendMessagesWithAnnotations(self, mock_worker_class, mock_conn_class): fs_message = send_args[0] packed_message_list = rdf_flows.PackedMessageList.protobuf() fs_message.data.Unpack(packed_message_list) - message_list = communicator.Communicator.DecompressMessageList( + message_list = _DecompressMessageList( rdf_flows.PackedMessageList.FromSerializedBytes( - packed_message_list.SerializeToString())) + packed_message_list.SerializeToString() + ) + ) self.assertListEqual(list(message_list.job), grr_messages) self.assertEqual(fs_message.annotations, expected_annotations) diff --git a/grr/client/grr_response_client/gcs.py b/grr/client/grr_response_client/gcs.py index ff825c49e1..9d65a25f84 100644 --- a/grr/client/grr_response_client/gcs.py +++ b/grr/client/grr_response_client/gcs.py @@ -1,5 +1,6 @@ #!/usr/bin/env python """Utilities for working with Google Cloud Storage.""" +import datetime import pathlib from typing import Callable from typing import IO @@ -62,9 +63,9 @@ class Opts: retry_chunk_attempts: A maximum number of attempts made when uploading a chunk. retry_chunk_init_delay: An initial delay value for retrying when uploading - a chunk (in seconds). + a chunk. retry_chunk_max_delay: A maximum delay value for retrying when uploading a - chunk (in seconds). + chunk. retry_chunk_backoff: A backoff multiplayer for extending the delay between chunk upload retries. progress_callback: A progress function to call periodically when uploading @@ -75,8 +76,8 @@ class Opts: chunk_size: int = 8 * 1024 * 1024 # 8 MiB. retry_chunk_attempts: int = 30 - retry_chunk_init_delay: float = 1.0 # 1 s. - retry_chunk_max_delay: float = 30 * 60.0 # 30 min. + retry_chunk_init_delay: datetime.timedelta = datetime.timedelta(seconds=1) + retry_chunk_max_delay: datetime.timedelta = datetime.timedelta(minutes=30) retry_chunk_backoff: float = 1.5 progress_callback: Callable[[], None] @@ -149,18 +150,20 @@ def SendFile(self, file: IO[bytes], opts: Optional[Opts] = None) -> None: if opts is None: opts = self.Opts() - def Sleep(secs: float) -> None: + def Sleep(timedelta: datetime.timedelta) -> None: time.Sleep( - secs, + timedelta.total_seconds(), progress_secs=opts.progress_interval, - progress_callback=opts.progress_callback) - - retry_opts = retry.Opts() - retry_opts.attempts = opts.retry_chunk_attempts - retry_opts.backoff = opts.retry_chunk_backoff - retry_opts.init_delay_secs = opts.retry_chunk_init_delay - retry_opts.max_delay_secs = opts.retry_chunk_max_delay - retry_opts.sleep = Sleep + progress_callback=opts.progress_callback, + ) + + retry_opts = retry.Opts( + attempts=opts.retry_chunk_attempts, + backoff=opts.retry_chunk_backoff, + init_delay=opts.retry_chunk_init_delay, + max_delay=opts.retry_chunk_max_delay, + sleep=Sleep, + ) offset = 0 diff --git a/grr/client/grr_response_client/gcs_test.py b/grr/client/grr_response_client/gcs_test.py index d20ef806a3..280decd57a 100644 --- a/grr/client/grr_response_client/gcs_test.py +++ b/grr/client/grr_response_client/gcs_test.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +import datetime import io import time from unittest import mock @@ -56,7 +57,7 @@ def testSendFileTransmissionFailure(self): opts = gcs.UploadSession.Opts() opts.retry_chunk_attempts = 1 - opts.retry_chunk_init_delay = 0.0 + opts.retry_chunk_init_delay = datetime.timedelta(0) with self.assertRaises(gcs.RequestError) as context: session.SendFile(io.BytesIO(b"foobar"), opts=opts) @@ -70,7 +71,7 @@ def testSendFileInterrupted(self): opts = gcs.UploadSession.Opts() opts.retry_chunk_attempts = 1 - opts.retry_chunk_init_delay = 0.0 + opts.retry_chunk_init_delay = datetime.timedelta(0) session = gcs.UploadSession("https://foo.bar/quux") @@ -159,7 +160,7 @@ def testSendFileRetrySuccess(self): opts = gcs.UploadSession.Opts() opts.chunk_size = 1 opts.retry_chunk_attempts = 4 - opts.retry_chunk_init_delay = 0.0 + opts.retry_chunk_init_delay = datetime.timedelta(0) session = gcs.UploadSession("https://foo.bar/qux") session.SendFile(io.BytesIO(b"foobar"), opts=opts) @@ -177,7 +178,7 @@ def testSendFileRetryFailure(self): opts = gcs.UploadSession.Opts() opts.chunk_size = 1 opts.retry_chunk_attempts = 3 - opts.retry_chunk_init_delay = 0.0 + opts.retry_chunk_init_delay = datetime.timedelta(0) session = gcs.UploadSession("https://foo.bar/qux") @@ -221,7 +222,7 @@ def Progress() -> None: opts = gcs.UploadSession.Opts() opts.retry_chunk_attempts = 2 - opts.retry_chunk_init_delay = 10.0 + opts.retry_chunk_init_delay = datetime.timedelta(seconds=10) opts.progress_interval = 1.0 opts.progress_callback = Progress diff --git a/grr/client/grr_response_client/poolclient.py b/grr/client/grr_response_client/poolclient.py deleted file mode 100644 index 5eaac79c25..0000000000 --- a/grr/client/grr_response_client/poolclient.py +++ /dev/null @@ -1,214 +0,0 @@ -#!/usr/bin/env python -"""This is the GRR client for thread pools.""" - -import base64 -import logging -import threading -import time - -from absl import app -from absl import flags - -# pylint: disable=unused-import -# Make sure we load the client plugins -from grr_response_client import client_plugins -# pylint: enable=unused-import - -from grr_response_client import client_startup -from grr_response_client import comms -from grr_response_client import vfs -from grr_response_client.vfs_handlers import files as vfs_files -from grr_response_core import config -from grr_response_core.config import contexts -from grr_response_core.lib import rdfvalue -from grr_response_core.lib.rdfvalues import crypto as rdf_crypto -from grr_response_core.lib.rdfvalues import paths as rdf_paths - -_NRCLIENTS = flags.DEFINE_integer("nrclients", 1, "Number of clients to start") - -_CERT_FILE = flags.DEFINE_string( - "cert_file", "", "Path to a file that stores all certificates for" - "the client pool.") - -_ENROLL_ONLY = flags.DEFINE_bool( - "enroll_only", False, - "If specified, the script will enroll all clients and exit.") - -_FAST_POLL = flags.DEFINE_bool( - "fast_poll", False, - "If specified, every client in the pool will work in the " - "fast poll mode. This is useful for benchmarks, as in fast " - "poll mode the timeouts are predictable and benchmarks " - "results are more stable.") - -_SEND_FOREMAN_REQUEST = flags.DEFINE_bool( - "send_foreman_request", False, - "If specified, every client will send a foreman poll request " - "right after startup. Useful for testing hunts.") - - -class PoolGRRClient(threading.Thread): - """A GRR client for running in pool mode.""" - - def __init__(self, - ca_cert=None, - private_key=None, - fast_poll=False, - send_foreman_request=False): - """Constructor.""" - super().__init__() - self.private_key = private_key - self.daemon = True - - self.client = comms.GRRHTTPClient(ca_cert=ca_cert, private_key=private_key) - if fast_poll: - self.client.timer.FastPoll() - - if send_foreman_request: - self.client.SendForemanRequest() - - self.stop = False - # Is this client already enrolled? - self.enrolled = False - - def Run(self): - while not self.stop: - status = self.client.RunOnce() - if status.code == 200: - self.enrolled = True - self.client.timer.Wait() - - def Stop(self): - self.stop = True - - def run(self): - self.Run() - - -def CreateClientPool(n): - """Create n clients to run in a pool.""" - clients = [] - - # Load previously stored clients. - try: - certificates = [] - with open(_CERT_FILE.value, "rb") as fd: - # Certificates are base64-encoded, so that we can use new-lines as - # separators. - for l in fd: - cert = rdf_crypto.RSAPrivateKey(initializer=base64.b64decode(l)) - certificates.append(cert) - - for certificate in certificates[:n]: - clients.append( - PoolGRRClient( - private_key=certificate, - ca_cert=config.CONFIG["CA.certificate"], - fast_poll=_FAST_POLL.value, - send_foreman_request=_SEND_FOREMAN_REQUEST.value, - )) - - clients_loaded = True - except (IOError, EOFError): - clients_loaded = False - - if clients_loaded and len(clients) < n: - raise RuntimeError( - "Loaded %d clients, but expected %d." % (len(clients), n)) - - while len(clients) < n: - # Generate a new RSA key pair for each client. - bits = config.CONFIG["Client.rsa_key_length"] - key = rdf_crypto.RSAPrivateKey.GenerateKey(bits=bits) - clients.append( - PoolGRRClient(private_key=key, ca_cert=config.CONFIG["CA.certificate"])) - - # Start all the clients now. - for c in clients: - c.start() - - start_time = rdfvalue.RDFDatetime.Now() - try: - if _ENROLL_ONLY.value: - while True: - time.sleep(1) - enrolled = len([x for x in clients if x.enrolled]) - - if enrolled == n: - logging.info("All clients enrolled, exiting.") - break - - else: - logging.info("%s: Enrolled %d/%d clients.", int(time.time()), - enrolled, n) - else: - try: - while True: - time.sleep(100) - except KeyboardInterrupt: - pass - - finally: - # Stop all pool clients. - for cl in clients: - cl.Stop() - - # Note: code below is going to be executed after SIGTERM is sent to this - # process. - duration = rdfvalue.RDFDatetime.Now() - start_time - logging.info("Pool done in %s seconds.", - duration.ToFractional(rdfvalue.SECONDS)) - - # The way benchmarking is supposed to work is that we execute poolclient with - # --enroll_only flag, it dumps the certificates to the flags.FLAGS.cert_file. - # Then, all further poolclient invocations just read private keys back - # from that file. Therefore if private keys were loaded from - # flags.FLAGS.cert_file, then there's no need to rewrite it again with the - # same data. - if not clients_loaded: - logging.info("Saving certificates.") - with open(_CERT_FILE.value, "wb") as fd: - # We're base64-encoding ceritificates so that we can use new-lines - # as separators. - b64_certs = [ - base64.b64encode(x.private_key.SerializeToBytes()) for x in clients - ] - fd.write("\n".join(b64_certs)) - - -def CheckLocation(): - """Checks that the poolclient is not accidentally ran against production.""" - for url in (config.CONFIG["Client.server_urls"] + - config.CONFIG["Client.control_urls"]): - if "staging" in url or "localhost" in url: - # This is ok. - return - logging.error("Poolclient should only be run against test or staging.") - exit() - - -class _UseOSForTSKFile(vfs_files.File): - """OS-file handler to be used for TSK files, since TSK is not threadsafe.""" - supported_pathtype = rdf_paths.PathSpec.PathType.TSK - - -def main(argv): - del argv # Unused. - config.CONFIG.AddContext(contexts.POOL_CLIENT_CONTEXT, - "Context applied when we run the pool client.") - - client_startup.ClientInit() - - config.CONFIG.SetWriteBack("/dev/null") - - CheckLocation() - - # Let the OS handler also handle sleuthkit requests since sleuthkit is not - # thread safe. - vfs.VFS_HANDLERS[_UseOSForTSKFile.supported_pathtype] = _UseOSForTSKFile - - CreateClientPool(_NRCLIENTS.value) - - -if __name__ == "__main__": - app.run(main) diff --git a/grr/client/grr_response_client/unprivileged/communication.py b/grr/client/grr_response_client/unprivileged/communication.py index db5917304f..230cd5df16 100644 --- a/grr/client/grr_response_client/unprivileged/communication.py +++ b/grr/client/grr_response_client/unprivileged/communication.py @@ -386,6 +386,7 @@ def Main(channel: Channel, connection_handler: ConnectionHandler, user: str, group: Unprivileged (UNIX) group to run as. If `""`, don't change group. """ sandbox.EnterSandbox(user, group) + assert channel.pipe_input is not None and channel.pipe_output is not None with os.fdopen( channel.pipe_input.ToFileDescriptor(), "rb", buffering=False) as pipe_input: diff --git a/grr/client/grr_response_client/unprivileged/communication_test.py b/grr/client/grr_response_client/unprivileged/communication_test.py index dc6366b393..b2242ec0d2 100644 --- a/grr/client/grr_response_client/unprivileged/communication_test.py +++ b/grr/client/grr_response_client/unprivileged/communication_test.py @@ -15,6 +15,7 @@ def _MakeArgs(channel: communication.Channel) -> List[str]: + assert channel.pipe_input is not None and channel.pipe_output is not None return [ sys.executable, "-m", "grr_response_client.unprivileged.echo_server", diff --git a/grr/client/grr_response_client/unprivileged/filesystem/server_lib.py b/grr/client/grr_response_client/unprivileged/filesystem/server_lib.py index 93b403caaa..9795594e5a 100644 --- a/grr/client/grr_response_client/unprivileged/filesystem/server_lib.py +++ b/grr/client/grr_response_client/unprivileged/filesystem/server_lib.py @@ -168,6 +168,7 @@ def HandleOperation( inode = request.inode if request.HasField('inode') else None stream_name = request.stream_name if request.HasField( 'stream_name') else None + assert state.filesystem is not None if inode is None: file_obj = state.filesystem.Open(path, stream_name) else: diff --git a/grr/client/grr_response_client/unprivileged/filesystem/vfs.py b/grr/client/grr_response_client/unprivileged/filesystem/vfs.py index c3b60586dd..2cf759ea5d 100644 --- a/grr/client/grr_response_client/unprivileged/filesystem/vfs.py +++ b/grr/client/grr_response_client/unprivileged/filesystem/vfs.py @@ -259,6 +259,7 @@ def Stat(self, def Read(self, length: int) -> bytes: self._CheckIsFile() + assert self.fd is not None data = self.fd.Read(self.offset, length) self.offset += len(data) return data @@ -271,6 +272,7 @@ def ListFiles(self, # pytype: disable=signature-mismatch # overriding-return-t del ext_attrs # Unused. self._CheckIsDirectory() + assert self.fd is not None for entry in self.fd.ListFiles(): pathspec = self.pathspec.Copy() @@ -283,6 +285,7 @@ def ListFiles(self, # pytype: disable=signature-mismatch # overriding-return-t def ListNames(self) -> Iterator[Text]: # pytype: disable=signature-mismatch # overriding-return-type-checks self._CheckIsDirectory() + assert self.fd is not None return iter(self.fd.ListNames()) def _CheckIsDirectory(self) -> None: @@ -295,12 +298,15 @@ def _CheckIsFile(self) -> None: raise IOError("{} is not a file".format(self.pathspec.CollapsePath())) def Close(self) -> None: + assert self.fd is not None self.fd.Close() def MatchBestComponentName( self, component: str, pathtype: rdf_paths.PathSpec) -> rdf_paths.PathSpec: fd = self.OpenAsContainer(pathtype) + assert self.fd is not None + new_component = self.fd.LookupCaseInsensitive(component) if new_component is not None: component = new_component diff --git a/grr/client/grr_response_client/unprivileged/server.py b/grr/client/grr_response_client/unprivileged/server.py index fe0f94d8ef..da92fa989b 100644 --- a/grr/client/grr_response_client/unprivileged/server.py +++ b/grr/client/grr_response_client/unprivileged/server.py @@ -12,7 +12,7 @@ def _MakeServerArgs(channel: communication.Channel, interface: interface_registry.Interface) -> List[str]: """Returns the args to run the unprivileged server command.""" - + assert channel.pipe_input is not None and channel.pipe_output is not None named_flags = [ "--unprivileged_server_pipe_input", str(channel.pipe_input.Serialize()), diff --git a/grr/client/grr_response_client/vfs_handlers/ntfs.py b/grr/client/grr_response_client/vfs_handlers/ntfs.py index 957ab7ae75..41bb66bf97 100644 --- a/grr/client/grr_response_client/vfs_handlers/ntfs.py +++ b/grr/client/grr_response_client/vfs_handlers/ntfs.py @@ -130,6 +130,7 @@ def Stat(self, def Read(self, length: int) -> bytes: self._CheckIsFile() + assert self.data_stream is not None self.data_stream.seek(self.offset) data = self.data_stream.read(length) self.offset += len(data) diff --git a/grr/client/grr_response_client/windows/installers.py b/grr/client/grr_response_client/windows/installers.py index e8761116d6..8b14f3bf70 100644 --- a/grr/client/grr_response_client/windows/installers.py +++ b/grr/client/grr_response_client/windows/installers.py @@ -23,17 +23,19 @@ import subprocess import sys import time -from typing import Callable, Iterable +from typing import Iterable +import winreg + from absl import flags import pywintypes import win32process import win32service import win32serviceutil import winerror -import winreg from grr_response_client.windows import regconfig from grr_response_core import config +from grr_response_core.lib.util import retry SERVICE_RESTART_DELAY_MSEC = 120 * 1000 @@ -198,9 +200,6 @@ def _StopPreviousService(): service_name=config.CONFIG["Nanny.service_name"], service_binary_name=config.CONFIG["Nanny.service_binary_name"]) - if not config.CONFIG["Client.fleetspeak_enabled"]: - return - _StopService(service_name=config.CONFIG["Client.fleetspeak_service_name"]) @@ -222,37 +221,30 @@ def _DeleteGrrFleetspeakService(): raise -def _FileRetryLoop(path: str, f: Callable[[], None]) -> None: - """If `path` exists, calls `f` in a retry loop.""" - if not os.path.exists(path): - return - attempts = 0 - while True: - try: - f() - return - except OSError as e: - attempts += 1 - if e.errno == errno.EACCES and attempts < 10: - # The currently installed GRR process may stick around for a few - # seconds after the service is terminated (keeping the contents of - # the installation directory locked). - logging.warning( - "Encountered permission-denied error while trying to process " - "'%s'. Retrying...", - path, - exc_info=True) - time.sleep(3) - else: - raise - - +@retry.When( + OSError, + lambda error: error.errno == errno.EACCES, + opts=retry.Opts( + attempts=10, + init_delay=datetime.timedelta(seconds=3), + ), +) def _RmTree(path: str) -> None: - _FileRetryLoop(path, lambda: shutil.rmtree(path)) - - + if os.path.exists(path): + shutil.rmtree(path) + + +@retry.When( + OSError, + lambda error: error.errno == errno.EACCES, + opts=retry.Opts( + attempts=10, + init_delay=datetime.timedelta(seconds=3), + ), +) def _Rename(src: str, dst: str) -> None: - _FileRetryLoop(src, lambda: os.rename(src, dst)) + if os.path.exists(src): + os.rename(src, dst) def _RmTreePseudoTransactional(path: str) -> None: @@ -343,29 +335,6 @@ def _CopyToSystemDir(): ["Nanny.status", "Nanny.heartbeat", "Client.labels"])) -def _InstallNanny(): - """Installs the nanny program.""" - # We need to copy the nanny sections to the registry to ensure the - # service is correctly configured. - new_config = config.CONFIG.MakeNewConfig() - new_config.SetWriteBack(config.CONFIG["Config.writeback"]) - - for option in _NANNY_OPTIONS: - new_config.Set(option, config.CONFIG.Get(option)) - - new_config.Write() - - args = [ - config.CONFIG["Nanny.binary"], "--service_key", - config.CONFIG["Client.config_key"], "install" - ] - - logging.debug("Calling %s", (args,)) - output = subprocess.check_output( - args, shell=True, stdin=subprocess.PIPE, stderr=subprocess.PIPE) - logging.debug("%s", output) - - def _DeleteLegacyConfigOptions(registry_key_uri): """Deletes config values in the registry for legacy GRR installations.""" key_spec = regconfig.ParseRegistryURI(registry_key_uri) @@ -485,11 +454,6 @@ def _Run(): _StartServices(STOPPED_SERVICES) raise - if not config.CONFIG["Client.fleetspeak_enabled"]: - logging.info("Fleetspeak not enabled, installing nanny.") - _InstallNanny() - return - # Remove the Nanny service for the legacy GRR since it will # not be needed any more. _RemoveService(config.CONFIG["Nanny.service_name"]) diff --git a/grr/client/setup.py b/grr/client/setup.py index 1e82b72079..41b48e0474 100644 --- a/grr/client/setup.py +++ b/grr/client/setup.py @@ -136,8 +136,7 @@ def run(self): "absl-py==1.4.0", "grr-response-core==%s" % VERSION.get("Version", "packagedepends"), "pytsk3==20230125", - "retry==0.9.2", - "libfsntfs-python==20221023", + "libfsntfs-python==20230606", "fleetspeak-client-bin==0.1.12", ], extras_require={ diff --git a/grr/core/grr_response_core/config/build.py b/grr/core/grr_response_core/config/build.py index 5b6071bf3b..4c5a671a80 100644 --- a/grr/core/grr_response_core/config/build.py +++ b/grr/core/grr_response_core/config/build.py @@ -57,12 +57,6 @@ "Directory where Fleetspeak expects service configs to be. Only applies " "if Client.fleetspeak_enabled is true.") -config_lib.DEFINE_string( - "ClientBuilder.fleetspeak_legacy_config", None, - "Path where previous versions of GRR installed Fleetspeak configs to. If " - "provided, the file at this location on the target system will be deleted " - "before GRR is installed.") - config_lib.DEFINE_string( "ClientBuilder.fleetspeak_plist_path", None, "Path where the Fleetspeak client installs its plist file. Only applies " diff --git a/grr/core/grr_response_core/config/client.py b/grr/core/grr_response_core/config/client.py index 160c56cdf6..74cb11a8ba 100644 --- a/grr/core/grr_response_core/config/client.py +++ b/grr/core/grr_response_core/config/client.py @@ -238,10 +238,6 @@ "%(Client.name)service.exe", help="The executable name of the nanny binary.") -config_lib.DEFINE_integer( - "Nanny.unresponsive_kill_period", 60, - "The time in seconds after which the nanny kills us.") - config_lib.DEFINE_integer( "Network.api", 3, "The version of the network protocol the client " "uses.") diff --git a/grr/core/grr_response_core/config/gui.py b/grr/core/grr_response_core/config/gui.py index 615f44a9df..73faa3f538 100644 --- a/grr/core/grr_response_core/config/gui.py +++ b/grr/core/grr_response_core/config/gui.py @@ -100,10 +100,18 @@ "%(Source.version_minor).%(Source.version_revision)", "Base path for GRR documentation. ") -config_lib.DEFINE_string( - "AdminUI.new_hunt_wizard.default_output_plugin", None, - "Output plugin that will be added by default in the " - "'New Hunt' wizard output plugins selection page.") + +# This accepts a comma-separated list of multiple plugins. Ideally, we'd use +# DEFINE_list instead of DEFINE_string, but returning lists as values of the +# config options is not supported by the GRR API (see ApiDataObjectKeyValuePair +# class and proto). +config_lib.DEFINE_string( + "AdminUI.new_hunt_wizard.default_output_plugins", + None, + "Output plugin(s) that will be added by default in the " + "'New Hunt' wizard output plugins selection page. Accepts comma-separated " + "list of multiple plugins.", +) config_lib.DEFINE_semantic_struct( rdf_config.AdminUIClientWarningsConfigOption, "AdminUI.client_warnings", diff --git a/grr/core/grr_response_core/lib/artifact_utils.py b/grr/core/grr_response_core/lib/artifact_utils.py index 4578665ee8..1639c8d43d 100644 --- a/grr/core/grr_response_core/lib/artifact_utils.py +++ b/grr/core/grr_response_core/lib/artifact_utils.py @@ -12,7 +12,6 @@ from grr_response_core.lib import interpolation -from grr_response_core.lib import objectfilter from grr_response_core.lib.rdfvalues import structs as rdf_structs @@ -266,27 +265,6 @@ def ExpandWindowsEnvironmentVariables(data_string, knowledge_base): return "".join(components) -def CheckCondition(condition, check_object): - """Check if a condition matches an object. - - Args: - condition: A string condition e.g. "os == 'Windows'" - check_object: Object to validate, e.g. an rdf_client.KnowledgeBase() - - Returns: - True or False depending on whether the condition matches. - - Raises: - ConditionError: If condition is bad. - """ - try: - of = objectfilter.Parser(condition).Parse() - compiled_filter = of.Compile(objectfilter.BaseFilterImplementation) - return compiled_filter.Matches(check_object) - except objectfilter.Error as e: - raise ConditionError(e) - - def ExpandWindowsUserEnvironmentVariables(data_string, knowledge_base, sid=None, diff --git a/grr/core/grr_response_core/lib/objectfilter.py b/grr/core/grr_response_core/lib/objectfilter.py deleted file mode 100644 index 0e66d4d823..0000000000 --- a/grr/core/grr_response_core/lib/objectfilter.py +++ /dev/null @@ -1,885 +0,0 @@ -#!/usr/bin/env python -# Copyright 2012 Google Inc. All Rights Reserved. -"""Classes to perform filtering of objects based on their data members. - -Given a list of objects and a textual filter expression, these classes allow -you to determine which objects match the filter. The system has two main -pieces: A parser for the supported grammar and a filter implementation. - -Given any complying user-supplied grammar, it is parsed with a custom lexer -based on GRR's lexer and then compiled into an actual implementation by using -the filter implementation. A filter implementation simply provides actual -implementations for the primitives required to perform filtering. The compiled -result is always a class supporting the Filter interface. - -If we define a class called Car such as: - -class Car(object): - def __init__(self, code, color="white", doors=3): - self.code = code - self.color = color - self.doors = 3 - -And we have two instances: - - ford_ka = Car("FORDKA1", color="grey") - toyota_corolla = Car("COROLLA1", color="white", doors=5) - fleet = [ford_ka, toyota_corolla] - -We want to find cars that are grey and have 3 or more doors. We could filter -our fleet like this: - - criteria = "(color is grey) and (doors >= 3)" - parser = ContextFilterParser(criteria).Parse() - compiled_filter = parser.Compile(LowercaseAttributeFilterImp) - - for car in fleet: - if compiled_filter.Matches(car): - print "Car %s matches the supplied filter." % car.code - -The filter expression contains two subexpressions joined by an AND operator: - "color is grey" and "doors >= 3" -This means we want to search for objects matching these two subexpressions. -Let's analyze the first one in depth "color is grey": - - "color": the left operand specifies a search path to look for the data. This - tells our filtering system to look for the color property on passed objects. - "is": the operator. Values retrieved for the "color" property will be checked - against the right operand to see if they are equal. - "grey": the right operand. It specifies an explicit value to check for. - -So each time an object is passed through the filter, it will expand the value -of the color data member, and compare its value against "grey". - -Because data members of objects are often not simple datatypes but other -objects, the system allows you to reference data members within other data -members by separating each by a dot. Let's see an example: - -Let's add a more complex Car class with default tyre data: - -class CarWithTyres(Car): - def __init__(self, code, tyres=None, color="white", doors=3): - super(self, CarWithTyres).__init__(code, color, doors) - tyres = tyres or Tyre("Pirelli", "PZERO") - -class Tyre(object): - def __init__(self, brand, code): - self.brand = brand - self.code = code - -And two new instances: - ford_ka = CarWithTyres("FORDKA", color="grey", tyres=Tyre("AVON", "ZT5")) - toyota_corolla = Car("COROLLA1", color="white", doors=5) - fleet = [ford_ka, toyota_corolla] - -To filter a car based on the tyre brand, we would use a search path of -"tyres.brand". - -Because the filter implementation provides the actual classes that perform -handling of the search paths, operators, etc. customizing the behaviour of the -filter is easy. Three basic filter implementations are given: - BaseFilterImplementation: search path expansion is done on attribute names - as provided (case-sensitive). - LowercaseAttributeFilterImp: search path expansion is done on the lowercased - attribute name, so that it only accesses attributes, not methods. - DictFilterImplementation: search path expansion is done on dictionary access - to the given object. So "a.b" expands the object obj to obj["a"]["b"] -""" - - -import abc -import binascii -from collections.abc import Mapping -import re -from typing import Text - - -from grr_response_core.lib import lexer -from grr_response_core.lib import utils -from grr_response_core.lib.util import precondition -from grr_response_core.lib.util import text - - -class Error(Exception): - """Base module exception.""" - - -class MalformedQueryError(Error): - """The provided filter query is malformed.""" - - -class ParseError(Error, lexer.ParseError): - """The parser for textual queries returned invalid results.""" - - -class InvalidNumberOfOperands(Error): - """The number of operands provided to this operator is wrong.""" - - -class Filter(metaclass=abc.ABCMeta): - """Base class for every filter.""" - - def __init__(self, arguments=None, value_expander=None): - """Constructor. - - Args: - arguments: Arguments to the filter. - value_expander: A callable that will be used to expand values for the - objects passed to this filter. Implementations expanders are provided by - subclassing ValueExpander. - - Raises: - Error: If the given value_expander is not a subclass of ValueExpander - """ - self.value_expander = None - self.value_expander_cls = value_expander - if self.value_expander_cls: - if not issubclass(self.value_expander_cls, ValueExpander): - raise Error( - "%s is not a valid value expander" % (self.value_expander_cls)) - self.value_expander = self.value_expander_cls() - self.args = arguments or [] - - @abc.abstractmethod - def Matches(self, obj): - """Whether object obj matches this filter.""" - - def Filter(self, objects): - """Returns a list of objects that pass the filter.""" - return list(filter(self.Matches, objects)) - - def __str__(self) -> Text: - return "%s(%s)" % (self.__class__.__name__, ", ".join( - [str(arg) for arg in self.args])) - - -class AndFilter(Filter): - """Performs a boolean AND of the given Filter instances as arguments. - - Note that if no conditions are passed, all objects will pass. - """ - - def Matches(self, obj): - for child_filter in self.args: - if not child_filter.Matches(obj): - return False - return True - - -class OrFilter(Filter): - """Performs a boolean OR of the given Filter instances as arguments. - - Note that if no conditions are passed, all objects will pass. - """ - - def Matches(self, obj): - if not self.args: - return True - for child_filter in self.args: - if child_filter.Matches(obj): - return True - return False - - -class Operator(Filter): - """Base class for all operators.""" - - -class IdentityFilter(Operator): - - def Matches(self, _): - return True - - -class UnaryOperator(Operator): - """Base class for unary operators.""" - - def __init__(self, operand, **kwargs): - """Constructor.""" - - super().__init__(arguments=[operand], **kwargs) - if len(self.args) != 1: - raise InvalidNumberOfOperands( - "Only one operand is accepted by %s. " - "Received %d." % (self.__class__.__name__, len(self.args))) - - -class BinaryOperator(Operator): - """Base class for binary operators. - - The left operand is always a path into the object which will be expanded for - values. The right operand is a value defined at initialization and is stored - at self.right_operand. - """ - - def __init__(self, arguments=None, **kwargs): - super().__init__(arguments=arguments, **kwargs) - if len(self.args) != 2: - raise InvalidNumberOfOperands( - "Only two operands are accepted by %s. " - "Received %d." % (self.__class__.__name__, len(self.args))) - self.left_operand = self.args[0] - self.right_operand = self.args[1] - - -class GenericBinaryOperator(BinaryOperator): - """Allows easy implementations of operators.""" - - def Operation(self, x, y): - """Performs the operation between two values.""" - - def Operate(self, values): - """Takes a list of values and if at least one matches, returns True.""" - for val in values: - try: - if self.Operation(val, self.right_operand): - return True - else: - continue - except (ValueError, TypeError): - continue - return False - - def Matches(self, obj): - key = self.left_operand - values = self.value_expander.Expand(obj, key) - if values and self.Operate(values): - return True - return False - - -class Equals(GenericBinaryOperator): - """Matches objects when the right operand equals the expanded value.""" - - def Operation(self, x, y): - return x == y - - -class NotEquals(GenericBinaryOperator): - """Matches when the right operand isn't equal to the expanded value.""" - - def Operate(self, values): - return not Equals( - arguments=self.args, - value_expander=self.value_expander_cls).Operate(values) - - -class Less(GenericBinaryOperator): - """Whether the expanded value >= right_operand.""" - - def Operation(self, x, y): - return x < y - - -class LessEqual(GenericBinaryOperator): - """Whether the expanded value <= right_operand.""" - - def Operation(self, x, y): - return x <= y - - -class Greater(GenericBinaryOperator): - """Whether the expanded value > right_operand.""" - - def Operation(self, x, y): - return x > y - - -class GreaterEqual(GenericBinaryOperator): - """Whether the expanded value >= right_operand.""" - - def Operation(self, x, y): - return x >= y - - -class Contains(GenericBinaryOperator): - """Whether the right operand is contained in the value.""" - - def Operation(self, x, y): - # Assuming x is iterable, check if it contains y. - # Otherwise, check if x and y are equal. - try: - return y in x - except TypeError: - return y == x - - -class NotContains(GenericBinaryOperator): - """Whether the right operand is not contained in the values.""" - - def Operate(self, values): - return not Contains( - arguments=self.args, - value_expander=self.value_expander_cls).Operate(values) - - -# TODO(user): Change to an N-ary Operator? -class InSet(GenericBinaryOperator): - """Whether all values are contained within the right operand.""" - - def Operation(self, x, y): - """Whether x is fully contained in y.""" - if x in y: - return True - - # x might be an iterable - # first we need to skip strings or we'll do silly things - if isinstance(x, str) or isinstance(x, bytes): - return False - - try: - for value in x: - if value not in y: - return False - return True - except TypeError: - return False - - -class NotInSet(GenericBinaryOperator): - """Whether at least a value is not present in the right operand.""" - - def Operate(self, values): - return not InSet( - arguments=self.args, - value_expander=self.value_expander_cls).Operate(values) - - -class Regexp(GenericBinaryOperator): - """Whether the value matches the regexp in the right operand.""" - - def __init__(self, *children, **kwargs): - super().__init__(*children, **kwargs) - try: - self.compiled_re = re.compile(utils.SmartUnicode(self.right_operand)) - except re.error: - raise ValueError( - "Regular expression \"%s\" is malformed." % self.right_operand) - - def Operation(self, x, y): - try: - if self.compiled_re.search(utils.SmartUnicode(x)): - return True - except TypeError: - return False - - -class Context(Operator): - """Restricts the child operators to a specific context within the object. - - Solves the context problem. The context problem is the following: - Suppose you store a list of loaded DLLs within a process. Suppose that for - each of these DLLs you store the number of imported functions and each of the - imported functions name. - - Imagine that a malicious DLL is injected into processes and its indicators are - that it only imports one function and that it is RegQueryValueEx. You'd write - your indicator like this: - - - AndOperator( - Equal("ImportedDLLs.ImpFunctions.Name", "RegQueryValueEx"), - Equal("ImportedDLLs.NumImpFunctions", "1") - ) - - Now imagine you have these two processes on a given system. - - Process1 - +[0]__ImportedDlls - +[0]__Name: "notevil.dll" - |[0]__ImpFunctions - | +[1]__Name: "CreateFileA" - |[0]__NumImpFunctions: 1 - | - +[1]__Name: "alsonotevil.dll" - |[1]__ImpFunctions - | +[0]__Name: "RegQueryValueEx" - | +[1]__Name: "CreateFileA" - |[1]__NumImpFunctions: 2 - - Process2 - +[0]__ImportedDlls - +[0]__Name: "evil.dll" - |[0]__ImpFunctions - | +[0]__Name: "RegQueryValueEx" - |[0]__NumImpFunctions: 1 - - Both Process1 and Process2 match your query, as each of the indicators are - evaluated separatedly. While you wanted to express "find me processes that - have a DLL that has both one imported function and ReqQueryValueEx is in the - list of imported functions", your indicator actually means "find processes - that have at least a DLL with 1 imported functions and at least one DLL that - imports the ReqQueryValueEx function". - - To write such an indicator you need to specify a context of ImportedDLLs for - these two clauses. Such that you convert your indicator to: - - Context("ImportedDLLs", - AndOperator( - Equal("ImpFunctions.Name", "RegQueryValueEx"), - Equal("NumImpFunctions", "1") - )) - - Context will execute the filter specified as the second parameter for each of - the objects under "ImportedDLLs", thus applying the condition per DLL, not per - object and returning the right result. - """ - - def __init__(self, arguments=None, **kwargs): - if len(arguments) != 2: - raise InvalidNumberOfOperands("Context accepts only 2 operands.") - super().__init__(arguments=arguments, **kwargs) - self.context, self.condition = self.args # pytype: disable=bad-unpacking - - def Matches(self, obj): - for object_list in self.value_expander.Expand(obj, self.context): - for sub_object in object_list: - if self.condition.Matches(sub_object): - return True - return False - - -OP2FN = { - "equals": Equals, - "is": Equals, - "==": Equals, - "notequals": NotEquals, - "isnot": NotEquals, - "!=": NotEquals, - "contains": Contains, - "notcontains": NotContains, - ">": Greater, - ">=": GreaterEqual, - "<": Less, - "<=": LessEqual, - "inset": InSet, - "notinset": NotInSet, - "regexp": Regexp -} - - -class ValueExpander(object): - """Encapsulates the logic to expand values available in an object. - - Once instantiated and called, this class returns all the values that follow a - given field path. - """ - - FIELD_SEPARATOR = "." - - def _GetAttributeName(self, path): - """Returns the attribute name to fetch given a path.""" - return path[0] - - def _GetValue(self, obj, attr_name): - """Returns the value of that attribute attr_name.""" - raise NotImplementedError() - - def _AtLeaf(self, attr_value): - """Called when at a leaf value. Should yield a value.""" - if isinstance(attr_value, Mapping): - # If the result is a dict, return each key/value pair as a new dict. - for k, v in attr_value.items(): - yield {k: v} - else: - yield attr_value - - def _AtNonLeaf(self, attr_value, path): - """Called when at a non-leaf value. Should recurse and yield values.""" - try: - if isinstance(attr_value, Mapping): - # If it's dictionary-like, treat the dict key as the attribute.. - sub_obj = attr_value.get(path[1]) - if len(path) > 2: - # Expand any additional elements underneath the key. - sub_obj = self.Expand(sub_obj, path[2:]) - if isinstance(sub_obj, str): - # If it is a string, stop here - yield sub_obj - elif isinstance(sub_obj, Mapping): - # If the result is a dict, return each key/value pair as a new dict. - for k, v in sub_obj.items(): - yield {k: v} - else: - for value in sub_obj: - yield value - else: - # If it's an iterable, we recurse on each value. - for sub_obj in attr_value: - for value in self.Expand(sub_obj, path[1:]): - yield value - except TypeError: # This is then not iterable, we recurse with the value - for value in self.Expand(attr_value, path[1:]): - yield value - - def Expand(self, obj, path): - """Returns a list of all the values for the given path in the object obj. - - Given a path such as ["sub1", "sub2"] it returns all the values available - in obj.sub1.sub2 as a list. sub1 and sub2 must be data attributes or - properties. - - If sub1 returns a list of objects, or a generator, Expand aggregates the - values for the remaining path for each of the objects, thus returning a - list of all the values under the given path for the input object. - - Args: - obj: An object that will be traversed for the given path - path: A list of strings - - Yields: - The values once the object is traversed. - """ - if isinstance(path, str): - path = path.split(self.FIELD_SEPARATOR) - - attr_name = self._GetAttributeName(path) - attr_value = self._GetValue(obj, attr_name) - if attr_value is None: - return - - if len(path) == 1: - for value in self._AtLeaf(attr_value): - yield value - else: - for value in self._AtNonLeaf(attr_value, path): - yield value - - -class AttributeValueExpander(ValueExpander): - """An expander that gives values based on object attribute names.""" - - def _GetValue(self, obj, attr_name): - if isinstance(obj, Mapping): - return obj.get(attr_name) - return getattr(obj, attr_name, None) - - -class LowercaseAttributeValueExpander(AttributeValueExpander): - """An expander that lowercases all attribute names before access.""" - - def _GetAttributeName(self, path): - return path[0].lower() - - -class DictValueExpander(ValueExpander): - """An expander that gets values from dictionary access to the object.""" - - def _GetValue(self, obj, attr_name): - return obj.get(attr_name, None) - - -# PARSER DEFINITION -class BasicExpression(lexer.Expression): - - def Compile(self, filter_implementation): - arguments = [self.attribute] - op_str = self.operator.lower() - operator = filter_implementation.OPS.get(op_str, None) - if not operator: - raise ParseError("Unknown operator %s provided." % self.operator) - arguments.extend(self.args) - expander = filter_implementation.FILTERS["ValueExpander"] - return operator(arguments=arguments, value_expander=expander) - - -class ContextExpression(lexer.Expression): - """Represents the context operator.""" - - def __init__(self, attribute="", part=None): - self.attribute = attribute - self.args = [] - if part: - self.args.append(part) - super().__init__() - - def __str__(self) -> Text: - return "Context(%s %s)" % (self.attribute, [str(x) for x in self.args]) - - def SetExpression(self, expression): - if isinstance(expression, lexer.Expression): - self.args = [expression] - else: - raise ParseError("Expected expression, got %s" % expression) - - def Compile(self, filter_implementation): - arguments = [self.attribute] - for arg in self.args: - arguments.append(arg.Compile(filter_implementation)) - expander = filter_implementation.FILTERS["ValueExpander"] - context_cls = filter_implementation.FILTERS["Context"] - return context_cls(arguments=arguments, value_expander=expander) - - -class BinaryExpression(lexer.BinaryExpression): - - def Compile(self, filter_implemention): - """Compile the binary expression into a filter object.""" - operator = self.operator.lower() - if operator == "and" or operator == "&&": - method = "AndFilter" - elif operator == "or" or operator == "||": - method = "OrFilter" - else: - raise ParseError("Invalid binary operator %s" % operator) - - args = [x.Compile(filter_implemention) for x in self.args] - return filter_implemention.FILTERS[method](arguments=args) - - -class IdentityExpression(lexer.Expression): - - def Compile(self, filter_implementation): - return filter_implementation.FILTERS["IdentityFilter"]() - - -class Parser(lexer.SearchParser): - """Parses and generates an AST for a query written in the described language. - - Examples of valid syntax: - size is 40 - (name contains "Program Files" AND hash.md5 is "123abc") - @imported_modules (num_symbols = 14 AND symbol.name is "FindWindow") - """ - expression_cls = BasicExpression - binary_expression_cls = BinaryExpression - context_cls = ContextExpression - identity_expression_cls = IdentityExpression - - list_args = [] - - tokens = [ - # Operators and related tokens - lexer.Token("INITIAL", r"\@[\w._0-9]+", "ContextOperator,PushState", - "CONTEXTOPEN"), - lexer.Token("INITIAL", r"[^\s\(\)]", "PushState,PushBack", "ATTRIBUTE"), - lexer.Token("INITIAL", r"\(", "PushState,BracketOpen", None), - lexer.Token("INITIAL", r"\)", "BracketClose", "BINARY"), - - # Context - lexer.Token("CONTEXTOPEN", r"\(", "BracketOpen", "INITIAL"), - - # Double quoted string - lexer.Token("STRING", "\"", "PopState,StringFinish", None), - lexer.Token("STRING", r"\\x(..)", "HexEscape", None), - lexer.Token("STRING", r"\\(.)", "StringEscape", None), - lexer.Token("STRING", r"[^\\\"]+", "StringInsert", None), - - # Single quoted string - lexer.Token("SQ_STRING", "'", "PopState,StringFinish", None), - lexer.Token("SQ_STRING", r"\\x(..)", "HexEscape", None), - lexer.Token("SQ_STRING", r"\\(.)", "StringEscape", None), - lexer.Token("SQ_STRING", r"[^\\']+", "StringInsert", None), - - # List processing. - lexer.Token("LIST_ARG", r"]", "PopState,ListFinish", None), - lexer.Token("LIST_ARG", r"(\d+\.\d+)", "InsertFloatArg", "LIST_ARG"), - lexer.Token("LIST_ARG", r"(0x[a-f\d]+)", "InsertInt16Arg", "LIST_ARG"), - lexer.Token("LIST_ARG", r"(\d+)", "InsertIntArg", "LIST_ARG"), - lexer.Token("LIST_ARG", "\"", "PushState,StringStart", "STRING"), - lexer.Token("LIST_ARG", "'", "PushState,StringStart", "SQ_STRING"), - lexer.Token("LIST_ARG", r",", None, None), - - # Basic expression - lexer.Token("ATTRIBUTE", r"[\w._0-9]+", "StoreAttribute", "OPERATOR"), - lexer.Token("OPERATOR", r"(\w+|[<>!=]=?)", "StoreOperator", "ARG"), - lexer.Token("ARG", r"(\d+\.\d+)", "InsertFloatArg", "ARG"), - lexer.Token("ARG", r"(0x[a-f\d]+)", "InsertInt16Arg", "ARG"), - lexer.Token("ARG", r"(\d+)", "InsertIntArg", "ARG"), - lexer.Token("ARG", r"\[", "PushState,ListStart", "LIST_ARG"), - lexer.Token("ARG", "\"", "PushState,StringStart", "STRING"), - lexer.Token("ARG", "'", "PushState,StringStart", "SQ_STRING"), - # When the last parameter from arg_list has been pushed - - # State where binary operators are supported (AND, OR) - lexer.Token("BINARY", r"(?i)(and|or|\&\&|\|\|)", "BinaryOperator", - "INITIAL"), - # - We can also skip spaces - lexer.Token("BINARY", r"\s+", None, None), - # - But if it's not "and" or just spaces we have to go back - lexer.Token("BINARY", ".", "PushBack,PopState", None), - - # Skip whitespace. - lexer.Token(".", r"\s+", None, None), - ] - - def InsertArg(self, string="", **_): - """Insert an arg to the current expression.""" - if self.state == "LIST_ARG": - self.list_args.append(string) - elif self.current_expression.AddArg(string): - # This expression is complete - self.stack.append(self.current_expression) - self.current_expression = self.expression_cls() - # We go to the BINARY state, to find if there's an AND or OR operator - return "BINARY" - - def InsertFloatArg(self, string="", **_): - """Inserts a Float argument.""" - try: - float_value = float(string) - return self.InsertArg(float_value) - except (TypeError, ValueError): - raise ParseError("%s is not a valid float." % string) - - def InsertIntArg(self, string="", **_): - """Inserts an Integer argument.""" - try: - int_value = int(string) - return self.InsertArg(int_value) - except (TypeError, ValueError): - raise ParseError("%s is not a valid integer." % string) - - def InsertInt16Arg(self, string="", **_): - """Inserts an Integer in base16 argument.""" - try: - int_value = int(string, 16) - return self.InsertArg(int_value) - except (TypeError, ValueError): - raise ParseError("%s is not a valid base16 integer." % string) - - def ListStart(self, **_): - self.list_args = [] - - def ListFinish(self, **_): - return self.InsertArg(string=self.list_args) - - def StringFinish(self, **_): - if self.state == "ATTRIBUTE": - return self.StoreAttribute(string=self.string) - - elif self.state == "ARG" or "LIST_ARG": - return self.InsertArg(string=self.string) - - def StringEscape(self, string, match, **_): - """Escape backslashes found inside a string quote. - - Backslashes followed by anything other than [\'"rnbt] will raise an Error. - - Args: - string: The string that matched. - match: The match object (m.group(1) is the escaped code) - - Raises: - ParseError: For strings other than those used to define a regexp, raise an - error if the escaped string is not one of [\'"rnbt]. - """ - precondition.AssertType(string, Text) - - # Allow unfiltered strings for regexp operations so that escaped special - # characters (e.g. \*) or special sequences (e.g. \w) can be used in - # objectfilter. - if self.current_expression.operator == "regexp": - self.string += text.Unescape(string) - elif match.group(1) in "\\'\"rnbt": - self.string += text.Unescape(string) - else: - raise ParseError("Invalid escape character %s." % string) - - def HexEscape(self, string, match, **_): - """Converts a hex escaped string.""" - hex_string = match.group(1) - try: - self.string += binascii.unhexlify(hex_string).decode("utf-8") - except binascii.Error as error: - raise ParseError("Invalid hex escape '{}': {}".format(hex_string, error)) - - def ContextOperator(self, string="", **_): - self.stack.append(self.context_cls(string[1:])) - - def Reduce(self): - """Reduce the token stack into an AST.""" - # Check for sanity - if self.state != "INITIAL" and self.state != "BINARY": - self.Error("Premature end of expression") - - length = len(self.stack) - while length > 1: - # Precedence order - self._CombineParenthesis() - self._CombineBinaryExpressions("and") - self._CombineBinaryExpressions("or") - self._CombineContext() - - # No change - if len(self.stack) == length: - break - length = len(self.stack) - - if length != 1: - self.Error("Illegal query expression") - - return self.stack[0] - - def Error(self, message=None, _=None): - raise ParseError("%s in position %s: %s <----> %s )" % - (message, len(self.processed_buffer), - self.processed_buffer, self.buffer)) - - def _CombineBinaryExpressions(self, operator): - for i in range(1, len(self.stack) - 1): - item = self.stack[i] - if (isinstance(item, lexer.BinaryExpression) and - item.operator.lower() == operator.lower() and - isinstance(self.stack[i - 1], lexer.Expression) and - isinstance(self.stack[i + 1], lexer.Expression)): - lhs = self.stack[i - 1] - rhs = self.stack[i + 1] - - self.stack[i].AddOperands(lhs, rhs) # pytype: disable=attribute-error - self.stack[i - 1] = None - self.stack[i + 1] = None - - self.stack = list(filter(None, self.stack)) - - def _CombineContext(self): - # Context can merge from item 0 - for i in range(len(self.stack) - 1, 0, -1): - item = self.stack[i - 1] - if (isinstance(item, ContextExpression) and - isinstance(self.stack[i], lexer.Expression)): - expression = self.stack[i] - item.SetExpression(expression) - self.stack[i] = None - - self.stack = list(filter(None, self.stack)) - - -# FILTER IMPLEMENTATIONS - - -class BaseFilterImplementation(object): - """Defines the base implementation of an object filter by its attributes. - - Inherit from this class, switch any of the needed operators and pass it to - the Compile method of a parsed string to obtain an executable filter. - """ - - OPS = OP2FN - FILTERS = { - "ValueExpander": AttributeValueExpander, - "AndFilter": AndFilter, - "OrFilter": OrFilter, - "IdentityFilter": IdentityFilter, - "Context": Context - } - - -class LowercaseAttributeFilterImplementation(BaseFilterImplementation): - """Does field name access on the lowercase version of names. - - Useful to only access attributes and properties with Google's python naming - style. - """ - - FILTERS = {} - FILTERS.update(BaseFilterImplementation.FILTERS) - FILTERS.update({"ValueExpander": LowercaseAttributeValueExpander}) - - -class DictFilterImplementation(BaseFilterImplementation): - """Does value fetching by dictionary access on the object.""" - - FILTERS = {} - FILTERS.update(BaseFilterImplementation.FILTERS) - FILTERS.update({"ValueExpander": DictValueExpander}) diff --git a/grr/core/grr_response_core/lib/objectfilter_test.py b/grr/core/grr_response_core/lib/objectfilter_test.py deleted file mode 100644 index cd16e91851..0000000000 --- a/grr/core/grr_response_core/lib/objectfilter_test.py +++ /dev/null @@ -1,528 +0,0 @@ -#!/usr/bin/env python -# Copyright 2012 Google Inc. All Rights Reserved. -"""Tests for grr.lib.objectfilter.""" - -from absl.testing import absltest - -from grr_response_core.lib import objectfilter - -attr1 = "Backup" -attr2 = "Archive" -hash1 = "123abc" -hash2 = "456def" -filename = "boot.ini" - - -class DummyObject(object): - - def __init__(self, key, value): - setattr(self, key, value) - - -class HashObject(object): - - def __init__(self, hash_value=None): - self.value = hash_value - - @property - def md5(self): - return self.value - - def __eq__(self, y): - return self.value == y - - def __ne__(self, other): - return not self.__eq__(other) - - def __lt__(self, y): - return self.value < y - - -class Dll(object): - - def __init__(self, name, imported_functions=None, exported_functions=None): - self.name = name - self._imported_functions = imported_functions or [] - self.num_imported_functions = len(self._imported_functions) - self.exported_functions = exported_functions or [] - self.num_exported_functions = len(self.exported_functions) - - @property - def imported_functions(self): - for fn in self._imported_functions: - yield fn - - -class DummyFile(object): - non_callable_leaf = "yoda" - - def __init__(self): - self.non_callable = HashObject(hash1) - self.non_callable_repeated = [ - DummyObject("desmond", ["brotha", "brotha"]), - DummyObject("desmond", ["brotha", "sista"]) - ] - self.imported_dll1 = Dll("a.dll", ["FindWindow", "CreateFileA"]) - self.imported_dll2 = Dll("b.dll", ["RegQueryValueEx"]) - - @property - def name(self): - return filename - - @property - def attributes(self): - return [attr1, attr2] - - @property - def hash(self): - return [HashObject(hash1), HashObject(hash2)] - - @property - def mapping(self): - return { - "hashes": [HashObject(hash1), HashObject(hash2)], - "nested": { - "attrs": [attr1, attr2] - }, - "string": "mate", - "float": 42.0 - } - - @property - def size(self): - return 10 - - @property - def deferred_values(self): - for v in ["a", "b"]: - yield v - - @property - def novalues(self): - return [] - - @property - def imported_dlls(self): - return [self.imported_dll1, self.imported_dll2] - - def Callable(self): - raise RuntimeError("This can not be called.") - - @property - def float(self): - return 123.9823 - - -class ObjectFilterTest(absltest.TestCase): - - def setUp(self): - self.file = DummyFile() - self.filter_imp = objectfilter.LowercaseAttributeFilterImplementation - self.value_expander = self.filter_imp.FILTERS["ValueExpander"] - - operator_tests = { - objectfilter.Less: [ - (True, ["size", 1000]), - (True, ["size", 11]), - (False, ["size", 10]), - (False, ["size", 0]), - (False, ["float", 1.0]), - (True, ["float", 123.9824]), - ], - objectfilter.LessEqual: [ - (True, ["size", 1000]), - (True, ["size", 11]), - (True, ["size", 10]), - (False, ["size", 9]), - (False, ["float", 1.0]), - (True, ["float", 123.9823]), - ], - objectfilter.Greater: [ - (True, ["size", 1]), - (True, ["size", 9.23]), - (False, ["size", 10]), - (False, ["size", 1000]), - (True, ["float", 122]), - (True, ["float", 1.0]), - ], - objectfilter.GreaterEqual: [ - (False, ["size", 1000]), - (False, ["size", 11]), - (True, ["size", 10]), - (True, ["size", 0]), - # Floats work fine too - (True, ["float", 122]), - (True, ["float", 123.9823]), - # Comparisons works with strings, although it might be a bit silly - (True, ["name", "aoot.ini"]), - ], - objectfilter.Contains: [ - # Contains works with strings - (True, ["name", "boot.ini"]), - (True, ["name", "boot"]), - (False, ["name", "meh"]), - # Works with generators - (True, ["imported_dlls.imported_functions", "FindWindow"]), - # But not with numbers - (False, ["size", 12]), - ], - objectfilter.NotContains: [ - (False, ["name", "boot.ini"]), - (False, ["name", "boot"]), - (True, ["name", "meh"]), - ], - objectfilter.Equals: [ - (True, ["name", "boot.ini"]), - (False, ["name", "foobar"]), - (True, ["float", 123.9823]), - ], - objectfilter.NotEquals: [ - (False, ["name", "boot.ini"]), - (True, ["name", "foobar"]), - (True, ["float", 25]), - ], - objectfilter.InSet: [ - (True, ["name", ["boot.ini", "autoexec.bat"]]), - (True, ["name", "boot.ini"]), - (False, ["name", "NOPE"]), - # All values of attributes are within these - (True, ["attributes", ["Archive", "Backup", "Nonexisting"]]), - # Not all values of attributes are within these - (False, ["attributes", ["Executable", "Sparse"]]), - ], - objectfilter.NotInSet: [ - (False, ["name", ["boot.ini", "autoexec.bat"]]), - (False, ["name", "boot.ini"]), - (True, ["name", "NOPE"]), - ], - objectfilter.Regexp: [ - (True, ["name", "^boot.ini$"]), - (True, ["name", "boot.ini"]), - (False, ["name", "^$"]), - (True, ["attributes", "Archive"]), - # One can regexp numbers if they're inclined - (True, ["size", 0]), - # But regexp doesn't work with lists or generators for the moment - (False, ["imported_dlls.imported_functions", "FindWindow"]) - ], - } - - def testBinaryOperators(self): - for operator, test_data in self.operator_tests.items(): - for test_unit in test_data: - print("Testing %s with %s and %s" % (operator, test_unit[0], - test_unit[1])) - kwargs = { - "arguments": test_unit[1], - "value_expander": self.value_expander - } - self.assertEqual(test_unit[0], operator(**kwargs).Matches(self.file)) - - def testExpand(self): - # Case insensitivity - values_lowercase = self.value_expander().Expand(self.file, "size") - values_uppercase = self.value_expander().Expand(self.file, "Size") - self.assertListEqual(list(values_lowercase), list(values_uppercase)) - - # Existing, non-repeated, leaf is a value - values = self.value_expander().Expand(self.file, "size") - self.assertListEqual(list(values), [10]) - - # Existing, non-repeated, leaf is a string in mapping - values = self.value_expander().Expand(self.file, "mapping.string") - self.assertListEqual(list(values), ["mate"]) - - # Existing, non-repeated, leaf is a scalar in mapping - values = self.value_expander().Expand(self.file, "mapping.float") - self.assertListEqual(list(values), [42.0]) - - # Existing, non-repeated, leaf is iterable - values = self.value_expander().Expand(self.file, "attributes") - self.assertListEqual(list(values), [[attr1, attr2]]) - - # Existing, repeated, leaf is value - values = self.value_expander().Expand(self.file, "hash.md5") - self.assertListEqual(list(values), [hash1, hash2]) - - # Existing, repeated, leaf is iterable - values = self.value_expander().Expand(self.file, - "non_callable_repeated.desmond") - self.assertListEqual( - list(values), [["brotha", "brotha"], ["brotha", "sista"]]) - - # Existing, repeated, leaf is mapping. - values = self.value_expander().Expand(self.file, "mapping.hashes") - self.assertListEqual(list(values), [hash1, hash2]) - values = self.value_expander().Expand(self.file, "mapping.nested.attrs") - self.assertListEqual(list(values), [[attr1, attr2]]) - - # Now with an iterator - values = self.value_expander().Expand(self.file, "deferred_values") - self.assertListEqual([list(value) for value in values], [["a", "b"]]) - - # Iterator > generator - values = self.value_expander().Expand(self.file, - "imported_dlls.imported_functions") - expected = [["FindWindow", "CreateFileA"], ["RegQueryValueEx"]] - self.assertListEqual([list(value) for value in values], expected) - - # Non-existing first path - values = self.value_expander().Expand(self.file, "nonexistent") - self.assertListEqual(list(values), []) - - # Non-existing in the middle - values = self.value_expander().Expand(self.file, "hash.mink.boo") - self.assertListEqual(list(values), []) - - # Non-existing as a leaf - values = self.value_expander().Expand(self.file, "hash.mink") - self.assertListEqual(list(values), []) - - # Non-callable leaf - values = self.value_expander().Expand(self.file, "non_callable_leaf") - self.assertListEqual(list(values), [DummyFile.non_callable_leaf]) - - # callable - values = self.value_expander().Expand(self.file, "Callable") - self.assertListEqual(list(values), []) - - # leaf under a callable. Will return nothing - values = self.value_expander().Expand(self.file, "Callable.a") - self.assertListEqual(list(values), []) - - def testGenericBinaryOperator(self): - - class TestBinaryOperator(objectfilter.GenericBinaryOperator): - values = list() - - def Operation(self, x, _): - return self.values.append(x) - - # Test a common binary operator - tbo = TestBinaryOperator( - arguments=["whatever", 0], value_expander=self.value_expander) - self.assertEqual(tbo.right_operand, 0) - self.assertEqual(tbo.args[0], "whatever") - tbo.Matches(DummyObject("whatever", "id")) - tbo.Matches(DummyObject("whatever", "id2")) - tbo.Matches(DummyObject("whatever", "bg")) - tbo.Matches(DummyObject("whatever", "bg2")) - self.assertListEqual(tbo.values, ["id", "id2", "bg", "bg2"]) - - def testContext(self): - self.assertRaises( - objectfilter.InvalidNumberOfOperands, - objectfilter.Context, - arguments=["context"], - value_expander=self.value_expander) - self.assertRaises( - objectfilter.InvalidNumberOfOperands, - objectfilter.Context, - arguments=[ - "context", - objectfilter.Equals( - arguments=["path", "value"], - value_expander=self.value_expander), - objectfilter.Equals( - arguments=["another_path", "value"], - value_expander=self.value_expander) - ], - value_expander=self.value_expander) - # "One imported_dll imports 2 functions AND one imported_dll imports - # function RegQueryValueEx" - arguments = [ - objectfilter.Equals(["imported_dlls.num_imported_functions", 1], - value_expander=self.value_expander), - objectfilter.Contains( - ["imported_dlls.imported_functions", "RegQueryValueEx"], - value_expander=self.value_expander) - ] - condition = objectfilter.AndFilter(arguments=arguments) - # Without context, it matches because both filters match separately - self.assertEqual(True, condition.Matches(self.file)) - - arguments = [ - objectfilter.Equals(["num_imported_functions", 2], - value_expander=self.value_expander), - objectfilter.Contains(["imported_functions", "RegQueryValueEx"], - value_expander=self.value_expander) - ] - condition = objectfilter.AndFilter(arguments=arguments) - # "The same DLL imports 2 functions AND one of these is RegQueryValueEx" - context = objectfilter.Context( - arguments=["imported_dlls", condition], - value_expander=self.value_expander) - # With context, it doesn't match because both don't match in the same dll - self.assertEqual(False, context.Matches(self.file)) - - # "One imported_dll imports only 1 function AND one imported_dll imports - # function RegQueryValueEx" - condition = objectfilter.AndFilter(arguments=[ - objectfilter.Equals( - arguments=["num_imported_functions", 1], - value_expander=self.value_expander), - objectfilter.Contains(["imported_functions", "RegQueryValueEx"], - value_expander=self.value_expander) - ]) - # "The same DLL imports 1 function AND it"s RegQueryValueEx" - context = objectfilter.Context(["imported_dlls", condition], - value_expander=self.value_expander) - self.assertEqual(True, context.Matches(self.file)) - - # Now test the context with a straight query - query = """ -@imported_dlls -( - imported_functions contains "RegQueryValueEx" - AND num_imported_functions == 1 -) -""" - filter_ = objectfilter.Parser(query).Parse() - filter_ = filter_.Compile(self.filter_imp) - self.assertEqual(True, filter_.Matches(self.file)) - - def testRegexpRaises(self): - self.assertRaises( - ValueError, - objectfilter.Regexp, - arguments=["name", "I [dont compile"], - value_expander=self.value_expander) - - def testEscaping(self): - parser = objectfilter.Parser(r"a is '\n'").Parse() - self.assertEqual(parser.args[0], "\n") - # Invalid escape sequence - parser = objectfilter.Parser(r"a is '\z'") - self.assertRaises(objectfilter.ParseError, parser.Parse) - # Can escape the backslash - parser = objectfilter.Parser(r"a is '\\'").Parse() - self.assertEqual(parser.args[0], "\\") - - # HEX ESCAPING - # This fails as it's not really a hex escaped string - parser = objectfilter.Parser(r"a is '\xJZ'") - self.assertRaises(objectfilter.ParseError, parser.Parse) - # Instead, this is what one should write - parser = objectfilter.Parser(r"a is '\\xJZ'").Parse() - self.assertEqual(parser.args[0], r"\xJZ") - # Standard hex-escape - parser = objectfilter.Parser(r"a is '\x41\x41\x41'").Parse() - self.assertEqual(parser.args[0], "AAA") - # Hex-escape + a character - parser = objectfilter.Parser(r"a is '\x414'").Parse() - self.assertEqual(parser.args[0], r"A4") - # How to include r'\x41' - parser = objectfilter.Parser(r"a is '\\x41'").Parse() - self.assertEqual(parser.args[0], r"\x41") - - def testParse(self): - # Arguments are either int, float or quoted string - objectfilter.Parser("attribute == 1").Parse() - objectfilter.Parser("attribute == 0x10").Parse() - objectfilter.Parser("attribute == 0xa").Parse() - objectfilter.Parser("attribute == 0xFF").Parse() - parser = objectfilter.Parser("attribute == 1a") - self.assertRaises(objectfilter.ParseError, parser.Parse) - objectfilter.Parser("attribute == 1.2").Parse() - objectfilter.Parser("attribute == 'bla'").Parse() - objectfilter.Parser("attribute == \"bla\"").Parse() - parser = objectfilter.Parser("something == red") - self.assertRaises(objectfilter.ParseError, parser.Parse) - - # Can't start with AND - parser = objectfilter.Parser("and something is 'Blue'") - self.assertRaises(objectfilter.ParseError, parser.Parse) - - # Need to close braces - objectfilter.Parser("(a is 3)").Parse() - parser = objectfilter.Parser("(a is 3") - self.assertRaises(objectfilter.ParseError, parser.Parse) - # Need to open braces to close them - parser = objectfilter.Parser("a is 3)") - self.assertRaises(objectfilter.ParseError, parser.Parse) - - # Can parse lists - objectfilter.Parser("attribute inset [1, 2, '3', 4.01, 0xa]").Parse() - # Need to close square braces for lists. - parser = objectfilter.Parser("attribute inset [1, 2, '3', 4.01, 0xA") - self.assertRaises(objectfilter.ParseError, parser.Parse) - # Need to opensquare braces to close lists. - parser = objectfilter.Parser("attribute inset 1, 2, '3', 4.01]") - self.assertRaises(objectfilter.ParseError, parser.Parse) - - # Context Operator alone is not accepted - parser = objectfilter.Parser("@attributes") - self.assertRaises(objectfilter.ParseError, parser.Parse) - # Accepted only with braces - objectfilter.Parser("@attributes( name is 'adrien')").Parse() - # Not without them - parser = objectfilter.Parser("@attributes name is 'adrien'") - self.assertRaises(objectfilter.ParseError, parser.Parse) - # Can nest context operators - query = "@imported_dlls( @imported_function( name is 'OpenFileA'))" - objectfilter.Parser(query).Parse() - # Can nest context operators and mix braces without it messing up - query = "@imported_dlls( @imported_function( name is 'OpenFileA'))" - parser = objectfilter.Parser(query).Parse() - query = """ -@imported_dlls -( - @imported_function - ( - name is 'OpenFileA' and ordinal == 12 - ) -) -""" - objectfilter.Parser(query).Parse() - # Mix context and binary operators - query = """ -@imported_dlls -( - @imported_function - ( - name is 'OpenFileA' - ) AND num_functions == 2 -) -""" - objectfilter.Parser(query).Parse() - - def testInset(self): - obj = DummyObject("clone", 2) - parser = objectfilter.Parser("clone inset [1, 2, 3]").Parse() - filter_ = parser.Compile(self.filter_imp) - self.assertEqual(filter_.Matches(obj), True) - obj = DummyObject("troubleshooter", "red") - parser = objectfilter.Parser("troubleshooter inset ['red', 'blue']").Parse() - filter_ = parser.Compile(self.filter_imp) - self.assertEqual(filter_.Matches(obj), True) - obj = DummyObject("troubleshooter", "infrared") - parser = objectfilter.Parser("troubleshooter inset ['red', 'blue']").Parse() - filter_ = parser.Compile(self.filter_imp) - self.assertEqual(filter_.Matches(obj), False) - - def testCompile(self): - obj = DummyObject("something", "Blue") - parser = objectfilter.Parser("something == 'Blue'").Parse() - filter_ = parser.Compile(self.filter_imp) - self.assertEqual(filter_.Matches(obj), True) - parser = objectfilter.Parser("something == 'Red'").Parse() - filter_ = parser.Compile(self.filter_imp) - self.assertEqual(filter_.Matches(obj), False) - parser = objectfilter.Parser("something == \"Red\"").Parse() - filter_ = parser.Compile(self.filter_imp) - self.assertEqual(filter_.Matches(obj), False) - obj = DummyObject("size", 4) - parser = objectfilter.Parser("size < 3").Parse() - filter_ = parser.Compile(self.filter_imp) - self.assertEqual(filter_.Matches(obj), False) - parser = objectfilter.Parser("size == 4").Parse() - filter_ = parser.Compile(self.filter_imp) - self.assertEqual(filter_.Matches(obj), True) - query = "something is 'Blue' and size notcontains 3" - parser = objectfilter.Parser(query).Parse() - filter_ = parser.Compile(self.filter_imp) - self.assertEqual(filter_.Matches(obj), False) - - -if __name__ == "__main__": - absltest.main() diff --git a/grr/core/grr_response_core/lib/rdfvalue.py b/grr/core/grr_response_core/lib/rdfvalue.py index 07f60b9d0a..6ad5e0af82 100644 --- a/grr/core/grr_response_core/lib/rdfvalue.py +++ b/grr/core/grr_response_core/lib/rdfvalue.py @@ -902,6 +902,10 @@ def ToFractional(self, timeunit: int) -> float: """ return self.microseconds / timeunit + def AsTimedelta(self) -> datetime.timedelta: + """Returns a standard `timedelta` object corresponding to the duration.""" + return datetime.timedelta(microseconds=self.ToInt(MICROSECONDS)) + @property def microseconds(self): return self._value diff --git a/grr/core/grr_response_core/lib/rdfvalue_test.py b/grr/core/grr_response_core/lib/rdfvalue_test.py index b022f592aa..a6addbca6a 100644 --- a/grr/core/grr_response_core/lib/rdfvalue_test.py +++ b/grr/core/grr_response_core/lib/rdfvalue_test.py @@ -497,6 +497,18 @@ def testComparisonIsEqualToIntegerComparison(self): if a != b: self.assertNotEqual(dur_a, dur_b) + def testAsTimedeltaMicrosecond(self): + duration = rdfvalue.Duration.From(1, rdfvalue.MICROSECONDS) + self.assertEqual(duration.AsTimedelta(), datetime.timedelta(microseconds=1)) + + def testAsTimedeltaSeconds(self): + duration = rdfvalue.Duration.From(42, rdfvalue.SECONDS) + self.assertEqual(duration.AsTimedelta(), datetime.timedelta(seconds=42)) + + def testAsTimedeltaDays(self): + duration = rdfvalue.Duration.From(1337, rdfvalue.DAYS) + self.assertEqual(duration.AsTimedelta(), datetime.timedelta(days=1337)) + class DocTest(test_lib.DocTest): module = rdfvalue diff --git a/grr/core/grr_response_core/lib/rdfvalues/file_finder.py b/grr/core/grr_response_core/lib/rdfvalues/file_finder.py index ed5e003ad7..452710779c 100644 --- a/grr/core/grr_response_core/lib/rdfvalues/file_finder.py +++ b/grr/core/grr_response_core/lib/rdfvalues/file_finder.py @@ -242,29 +242,6 @@ class FileFinderResult(rdf_structs.RDFProtoStruct): ] -class CollectSingleFileArgs(rdf_structs.RDFProtoStruct): - """Arguments for CollectSingleFile.""" - protobuf = flows_pb2.CollectSingleFileArgs - rdf_deps = [rdfvalue.ByteSize] - - -class CollectSingleFileResult(rdf_structs.RDFProtoStruct): - """Result returned by CollectSingleFile.""" - protobuf = flows_pb2.CollectSingleFileResult - rdf_deps = [ - rdf_crypto.Hash, - rdf_client_fs.StatEntry, - ] - - -class CollectSingleFileProgress(rdf_structs.RDFProtoStruct): - """Progress returned by CollectSingleFile.""" - protobuf = flows_pb2.CollectSingleFileProgress - rdf_deps = [ - CollectSingleFileResult, - ] - - class CollectFilesByKnownPathArgs(rdf_structs.RDFProtoStruct): """Arguments for CollectFilesByKnownPath.""" protobuf = flows_pb2.CollectFilesByKnownPathArgs @@ -314,3 +291,42 @@ class CollectMultipleFilesProgress(rdf_structs.RDFProtoStruct): """Progress returned by CollectMultipleFiles.""" protobuf = flows_pb2.CollectMultipleFilesProgress rdf_deps = [] + + +class StatMultipleFilesArgs(rdf_structs.RDFProtoStruct): + """Arguments for StatMultipleFiles.""" + + protobuf = flows_pb2.StatMultipleFilesArgs + rdf_deps = [ + rdf_paths.GlobExpression, + FileFinderModificationTimeCondition, + FileFinderAccessTimeCondition, + FileFinderInodeChangeTimeCondition, + FileFinderSizeCondition, + FileFinderExtFlagsCondition, + FileFinderContentsRegexMatchCondition, + FileFinderContentsLiteralMatchCondition, + ] + + +class HashMultipleFilesArgs(rdf_structs.RDFProtoStruct): + """Arguments for HashMultipleFiles.""" + + protobuf = flows_pb2.HashMultipleFilesArgs + rdf_deps = [ + rdf_paths.GlobExpression, + FileFinderModificationTimeCondition, + FileFinderAccessTimeCondition, + FileFinderInodeChangeTimeCondition, + FileFinderSizeCondition, + FileFinderExtFlagsCondition, + FileFinderContentsRegexMatchCondition, + FileFinderContentsLiteralMatchCondition, + ] + + +class HashMultipleFilesProgress(rdf_structs.RDFProtoStruct): + """Progress returned by HashMultipleFiles.""" + + protobuf = flows_pb2.HashMultipleFilesProgress + rdf_deps = [] diff --git a/grr/core/grr_response_core/lib/util/retry.py b/grr/core/grr_response_core/lib/util/retry.py index 4a0042df5f..b505e20eca 100644 --- a/grr/core/grr_response_core/lib/util/retry.py +++ b/grr/core/grr_response_core/lib/util/retry.py @@ -1,47 +1,96 @@ #!/usr/bin/env python """A module with utilities for retrying function execution.""" +import dataclasses +import datetime import functools import logging -import math +import random import time from typing import Callable +from typing import Generic from typing import Optional from typing import Tuple from typing import Type from typing import TypeVar +from typing import Union +@dataclasses.dataclass class Opts: """Options that customize the retry mechanism. Attributes: attempts: The number of attempts to retry the call. - init_delay_secs: An initial value for delay between retries. - max_delay_secs: A maximum value for delay between retries. + init_delay: An initial value for delay between retries. + max_delay: A maximum value for delay between retries. backoff: A backoff multiplayer for the delay between retries. + jitter: A random jitter to add to delay between retries. sleep: A sleep function used for delaying retries. """ attempts: int = 1 - init_delay_secs: float = 0.0 - max_delay_secs: float = math.inf + init_delay: datetime.timedelta = datetime.timedelta(0) + max_delay: datetime.timedelta = datetime.timedelta.max backoff: float = 1.0 + jitter: float = 0.0 - sleep: Callable[[float], None] = time.sleep + # pyformat: disable + sleep: Callable[[datetime.timedelta], None] = ( + lambda timedelta: time.sleep(timedelta.total_seconds()) + ) + # pyformat: enable -class On: +_E = TypeVar("_E", bound=Exception) + + +class On(Generic[_E]): """A decorator that retries the wrapped function on exception.""" def __init__( self, - exceptions: Tuple[Type[Exception], ...], + exception: Union[Type[_E], Tuple[Type[_E], ...]], opts: Optional[Opts] = None, ) -> None: """Initializes the decorator. Args: - exceptions: A sequence of exceptions to retry on. + exception: A sequence of exceptions to retry on. + opts: Options that customize the retry behaviour. + """ + self._when = When(exception, lambda _: True, opts=opts) + + _R = TypeVar("_R") + + def __call__(self, func: Callable[..., _R]) -> Callable[..., _R]: + """Wraps the specified function into a retryable function. + + The wrapped function will be attempted to be called specified number of + times after which the error will be propagated. + + Args: + func: A function to wrap. + + Returns: + A wrapped function that retries on failures. + """ + return self._when(func) + + +class When(Generic[_E]): + """A decorator that retries function on exception if predicate is met.""" + + def __init__( + self, + exception: Union[Type[_E], Tuple[Type[_E], ...]], + predicate: Callable[[_E], bool], + opts: Optional[Opts] = None, + ) -> None: + """Initializes the decorator. + + Args: + exception: An exception type to catch. + predicate: A predicate to check whether to retry the exception. opts: Options that customize the retry behaviour. """ if opts is None: @@ -50,14 +99,13 @@ def __init__( if opts.attempts < 1: raise ValueError("Non-positive number of retries") - self._exceptions = exceptions + self._exception = exception + self._predicate = predicate self._opts = opts _R = TypeVar("_R") - # TODO(hanuszczak): Looks like there is a bug in the linter: it recognizes - # `_R` in the argument but doesn't recognize it in the result type position. - def __call__(self, func: Callable[..., _R]) -> Callable[..., _R]: # pylint: disable=undefined-variable + def __call__(self, func: Callable[..., _R]) -> Callable[..., _R]: """Wraps the specified function into a retryable function. The wrapped function will be attempted to be called specified number of @@ -74,19 +122,28 @@ def __call__(self, func: Callable[..., _R]) -> Callable[..., _R]: # pylint: dis @functools.wraps(func) def Wrapped(*args, **kwargs) -> On._R: attempts = 0 - delay_secs = opts.init_delay_secs + delay = opts.init_delay while True: try: return func(*args, **kwargs) - except self._exceptions as error: + except self._exception as error: attempts += 1 if attempts == opts.attempts: raise - logging.warning("'%s', to be retried in %s s.", error, delay_secs) + if not self._predicate(error): + raise + + jitter = random.uniform(-opts.jitter, +opts.jitter) + jittered_delay = delay + delay * jitter + + logging.warning("'%s', to be retried in %s", error, jittered_delay) + opts.sleep(jittered_delay) - opts.sleep(delay_secs) - delay_secs = min(delay_secs * opts.backoff, opts.max_delay_secs) + # Note that we calculate the new delay value basing on the base delay, + # not the jittered one, otherwise the delay values might become too + # unpredictable. + delay = min(delay * opts.backoff, opts.max_delay) return Wrapped diff --git a/grr/core/grr_response_core/lib/util/retry_test.py b/grr/core/grr_response_core/lib/util/retry_test.py index 6512a45b00..0daf576d4f 100644 --- a/grr/core/grr_response_core/lib/util/retry_test.py +++ b/grr/core/grr_response_core/lib/util/retry_test.py @@ -1,4 +1,6 @@ #!/usr/bin/env python +import datetime + from absl.testing import absltest from grr_response_core.lib.util import retry @@ -7,16 +9,18 @@ class OnTest(absltest.TestCase): def testNegativeTries(self): - opts = retry.Opts() - opts.attempts = -1 + opts = retry.Opts( + attempts=-1, + ) with self.assertRaisesRegex(ValueError, "number of retries"): retry.On((), opts=opts) def testImmediateSuccess(self): - opts = retry.Opts() - opts.attempts = 1 - opts.sleep = lambda _: None + opts = retry.Opts( + attempts=1, + sleep=lambda _: None, + ) @retry.On((), opts=opts) def Func() -> None: @@ -25,13 +29,14 @@ def Func() -> None: Func() # Should not raise. def testRetriedSuccess(self): - opts = retry.Opts() - opts.attempts = 3 - opts.sleep = lambda _: None + opts = retry.Opts( + attempts=3, + sleep=lambda _: None, + ) counter = [] - @retry.On((RuntimeError,), opts=opts) + @retry.On(RuntimeError, opts=opts) def Func() -> None: counter.append(()) if len(counter) < 3: @@ -40,13 +45,14 @@ def Func() -> None: Func() # Should not raise. def testRetriedFailure(self): - opts = retry.Opts() - opts.attempts = 3 - opts.sleep = lambda _: None + opts = retry.Opts( + attempts=3, + sleep=lambda _: None, + ) counter = [] - @retry.On((RuntimeError,), opts=opts) + @retry.On(RuntimeError, opts=opts) def Func() -> None: counter.append(()) if len(counter) < 4: @@ -58,57 +64,155 @@ def Func() -> None: def testBackoff(self): delays = [] - opts = retry.Opts() - opts.attempts = 7 - opts.init_delay_secs = 1.0 - opts.backoff = 2.0 - opts.sleep = delays.append + opts = retry.Opts( + attempts=7, + init_delay=datetime.timedelta(seconds=1), + backoff=2.0, + sleep=delays.append, + ) - @retry.On((RuntimeError,), opts=opts) + @retry.On(RuntimeError, opts=opts) def Func() -> None: raise RuntimeError() with self.assertRaises(RuntimeError): Func() - self.assertEqual(delays, [1.0, 2.0, 4.0, 8.0, 16.0, 32.0]) + self.assertEqual( + delays, + [ + datetime.timedelta(seconds=1), + datetime.timedelta(seconds=2), + datetime.timedelta(seconds=4), + datetime.timedelta(seconds=8), + datetime.timedelta(seconds=16), + datetime.timedelta(seconds=32), + ], + ) + + def testJitter(self): + delays = [] + + opts = retry.Opts( + attempts=4, + init_delay=datetime.timedelta(seconds=1), + jitter=0.5, + sleep=delays.append, + ) + + @retry.On(RuntimeError, opts=opts) + def Func() -> None: + raise RuntimeError() + + with self.assertRaises(RuntimeError): + Func() + + self.assertLen(delays, 3) + self.assertBetween( + delays[0], + datetime.timedelta(seconds=0.5), + datetime.timedelta(seconds=1.5), + ) + self.assertBetween( + delays[1], + datetime.timedelta(seconds=0.5), + datetime.timedelta(seconds=1.5), + ) + self.assertBetween( + delays[2], + datetime.timedelta(seconds=0.5), + datetime.timedelta(seconds=1.5), + ) + + def testJitterAndBackoff(self): + delays = [] + + opts = retry.Opts( + attempts=4, + init_delay=datetime.timedelta(seconds=1), + backoff=2.0, + jitter=0.5, + sleep=delays.append, + ) + + @retry.On(RuntimeError, opts=opts) + def Func() -> None: + raise RuntimeError() + + with self.assertRaises(RuntimeError): + Func() + + self.assertLen(delays, 3) + self.assertBetween( + delays[0], + datetime.timedelta(seconds=0.5), + datetime.timedelta(seconds=1.5), + ) + self.assertBetween( + delays[1], + datetime.timedelta(seconds=1.0), + datetime.timedelta(seconds=4.0), + ) + self.assertBetween( + delays[2], + datetime.timedelta(seconds=2.0), + datetime.timedelta(seconds=8.0), + ) def testInitDelay(self): delays = [] - opts = retry.Opts() - opts.attempts = 4 - opts.init_delay_secs = 42.0 - opts.backoff = 1.0 - opts.sleep = delays.append + opts = retry.Opts( + attempts=4, + init_delay=datetime.timedelta(seconds=42), + backoff=1.0, + sleep=delays.append, + ) - @retry.On((RuntimeError,), opts=opts) + @retry.On(RuntimeError, opts=opts) def Func() -> None: raise RuntimeError() with self.assertRaises(RuntimeError): Func() - self.assertEqual(delays, [42.0, 42.0, 42.0]) + self.assertEqual( + delays, + [ + datetime.timedelta(seconds=42), + datetime.timedelta(seconds=42), + datetime.timedelta(seconds=42), + ], + ) def testMaxDelay(self): delays = [] - opts = retry.Opts() - opts.attempts = 6 - opts.init_delay_secs = 1.0 - opts.max_delay_secs = 3.0 - opts.backoff = 1.5 - opts.sleep = delays.append + opts = retry.Opts( + attempts=6, + init_delay=datetime.timedelta(seconds=1), + max_delay=datetime.timedelta(seconds=3), + backoff=1.5, + sleep=delays.append, + ) - @retry.On((RuntimeError,), opts=opts) + @retry.On(RuntimeError, opts=opts) def Func() -> None: raise RuntimeError() with self.assertRaises(RuntimeError): Func() - self.assertEqual(delays, [1.0, 1.5, 2.25, 3.0, 3.0]) + self.assertEqual( + delays, + [ + datetime.timedelta(seconds=1), + datetime.timedelta(seconds=1.5), + datetime.timedelta(seconds=2.25), + datetime.timedelta(seconds=3), + datetime.timedelta(seconds=3), + ], + ) def testImmediateArgumentsAndResult(self): @@ -119,13 +223,14 @@ def Func(left: str, right: str) -> int: self.assertEqual(Func("13", "37"), 1337) def testRetriedArgumentAndResult(self): - opts = retry.Opts() - opts.attempts = 3 - opts.sleep = lambda _: None + opts = retry.Opts( + attempts=3, + sleep=lambda _: None, + ) counter = [] - @retry.On((RuntimeError,), opts=opts) + @retry.On(RuntimeError, opts=opts) def Func(left: str, right: str) -> int: counter.append(()) if len(counter) < 3: @@ -161,9 +266,10 @@ class BarError(Exception): class BazError(Exception): pass - opts = retry.Opts() - opts.attempts = 4 - opts.sleep = lambda _: None + opts = retry.Opts( + attempts=4, + sleep=lambda _: None, + ) counter = [] @@ -180,5 +286,114 @@ def Func() -> None: Func() # Should not raise. +class WhenTest(absltest.TestCase): + + def testImmediateSuccess(self): + opts = retry.Opts( + attempts=1, + sleep=lambda _: None, + ) + + @retry.When(RuntimeError, lambda _: False, opts=opts) + def Func() -> None: + pass + + Func() # Should not raise. + + def testRetriedSuccess(self): + opts = retry.Opts( + attempts=3, + sleep=lambda _: None, + ) + + counter = [] + + @retry.When(RuntimeError, lambda _: True, opts=opts) + def Func() -> None: + counter.append(()) + if len(counter) < 3: + raise RuntimeError() + + Func() # Should not raise. + + def testRetriedFailure(self): + opts = retry.Opts( + attempts=3, + sleep=lambda _: None, + ) + + counter = [] + + @retry.When(RuntimeError, lambda _: True, opts=opts) + def Func() -> None: + counter.append(()) + if len(counter) < 4: + raise RuntimeError() + + with self.assertRaises(RuntimeError): + Func() + + def testNegativePredicate(self): + opts = retry.Opts( + attempts=3, + sleep=lambda _: None, + ) + + counter = [] + + @retry.When(RuntimeError, lambda _: False, opts=opts) + def Func() -> None: + counter.append(()) + raise RuntimeError() + + with self.assertRaises(RuntimeError): + Func() + + self.assertLen(counter, 1) + + def testUnexpectedException(self): + class FooError(Exception): + pass + + class BarError(Exception): + pass + + @retry.When(FooError, lambda _: True) + def Func() -> None: + raise BarError() + + with self.assertRaises(BarError): + Func() + + def testMultipleExceptions(self): + class FooError(Exception): + pass + + class BarError(Exception): + pass + + class BazError(Exception): + pass + + opts = retry.Opts( + attempts=4, + sleep=lambda _: None, + ) + + counter = [] + + @retry.When((FooError, BarError, BazError), lambda _: True, opts=opts) + def Func() -> None: + counter.append(()) + if counter == 1: + raise FooError() + if counter == 2: + raise BarError() + if counter == 3: + raise BazError() + + Func() # Should not raise. + + if __name__ == "__main__": absltest.main() diff --git a/grr/core/install_data/etc/grr-server.yaml b/grr/core/install_data/etc/grr-server.yaml index 413da793a9..38a0f7c1aa 100644 --- a/grr/core/install_data/etc/grr-server.yaml +++ b/grr/core/install_data/etc/grr-server.yaml @@ -90,7 +90,6 @@ Target:Darwin: Client.poll_max: 5 Client.foreman_check_frequency: 30 Client.rss_max: 4000 - Nanny.unresponsive_kill_period: 3600 Client.prefix: dbg_ Target:Linux: @@ -152,7 +151,6 @@ Target:Linux: Client.poll_max: 5 Client.foreman_check_frequency: 30 Client.rss_max: 4000 - Nanny.unresponsive_kill_period: 3600 Client.prefix: dbg_ Target:Windows: @@ -191,7 +189,6 @@ Target:Windows: Client.poll_max: 5 Client.foreman_check_frequency: 30 Client.rss_max: 4000 - Nanny.unresponsive_kill_period: 3600 Client.prefix: dbg_ ConfigUpdater Context: @@ -210,6 +207,8 @@ Platform:Darwin: Logging.syslog_path: /var/run/syslog + Logging.filename: "%(Logging.path)/%(Client.name).log" + Platform:Linux: Client.binary_name: grrd @@ -221,6 +220,8 @@ Platform:Linux: Logging.syslog_path: /dev/log + Logging.filename: "%(Logging.path)/%(Client.name).log" + Commandline Context: Logging.engines: stderr @@ -271,5 +272,8 @@ Worker Context: Cron.active: True Logging.filename: "%(Logging.path)/grr-worker.log" +FleetspeakFrontend Context: + Logging.filename: "%(Logging.path)/grr-fleetspeak-frontend.log" + # For Test Context, see test_data/grr_test.yaml in the grr-response-test # package. diff --git a/grr/core/install_data/fleetspeak/darwin/grr_service_config.txt.in b/grr/core/install_data/fleetspeak/darwin/grr_service_config.txt.in index 745d780ab2..ee10df9bc4 100644 --- a/grr/core/install_data/fleetspeak/darwin/grr_service_config.txt.in +++ b/grr/core/install_data/fleetspeak/darwin/grr_service_config.txt.in @@ -8,5 +8,9 @@ config { [type.googleapis.com/fleetspeak.daemonservice.Config] { argv: "%(ClientBuilder.install_dir)/%(Client.binary_name)" argv: "--config=%(ClientBuilder.install_dir)/%(ClientBuilder.config_filename)" + + monitor_heartbeats: true + heartbeat_unresponsive_grace_period_seconds: 600 # 10 minutes. + heartbeat_unresponsive_kill_period_seconds: 120 # 2 minutes. } } diff --git a/grr/core/install_data/fleetspeak/linux/grr_service_config.txt.in b/grr/core/install_data/fleetspeak/linux/grr_service_config.txt.in index b531dce712..2d12ac3f01 100644 --- a/grr/core/install_data/fleetspeak/linux/grr_service_config.txt.in +++ b/grr/core/install_data/fleetspeak/linux/grr_service_config.txt.in @@ -8,5 +8,9 @@ config { [type.googleapis.com/fleetspeak.daemonservice.Config] { argv: "%(ClientBuilder.daemon_link)" argv: "--config=%(ClientBuilder.target_dir)/%(ClientBuilder.config_filename)" + + monitor_heartbeats: true + heartbeat_unresponsive_grace_period_seconds: 600 # 10 minutes. + heartbeat_unresponsive_kill_period_seconds: 120 # 2 minutes. } } diff --git a/grr/core/install_data/fleetspeak/windows/grr_service_config.txt.in b/grr/core/install_data/fleetspeak/windows/grr_service_config.txt.in index 4a72a52e5f..46eb43a309 100644 --- a/grr/core/install_data/fleetspeak/windows/grr_service_config.txt.in +++ b/grr/core/install_data/fleetspeak/windows/grr_service_config.txt.in @@ -13,7 +13,7 @@ config { memory_limit: 2147483648 # 2GB monitor_heartbeats: true - heartbeat_unresponsive_grace_period_seconds: 600 - heartbeat_unresponsive_kill_period_seconds: %(Nanny.unresponsive_kill_period) + heartbeat_unresponsive_grace_period_seconds: 600 # 10 minutes. + heartbeat_unresponsive_kill_period_seconds: 120 # 2 minutes. } } diff --git a/grr/core/install_data/macosx/client/fleetspeak/preinstall.sh.in b/grr/core/install_data/macosx/client/fleetspeak/preinstall.sh.in index 734ca23ee5..869bd08db3 100644 --- a/grr/core/install_data/macosx/client/fleetspeak/preinstall.sh.in +++ b/grr/core/install_data/macosx/client/fleetspeak/preinstall.sh.in @@ -19,9 +19,3 @@ if [[ -e '%(ClientBuilder.install_dir)' ]]; then tmpdir=`mktemp -d -t '%(Client.name)_%(Source.version_string)_install'` sudo mv '%(ClientBuilder.install_dir)' "${tmpdir}" fi - -# We do not want Fleetspeak to run two GRR instances on the same machine, -# so we delete any previous configs if they exist. -if [[ -e '%(ClientBuilder.fleetspeak_legacy_config)' ]]; then - rm -f '%(ClientBuilder.fleetspeak_legacy_config)' -fi diff --git a/grr/core/setup.py b/grr/core/setup.py index 03de362bd4..387fe948ea 100644 --- a/grr/core/setup.py +++ b/grr/core/setup.py @@ -135,7 +135,7 @@ def make_release_tree(self, base_dir, files): "python-crontab==2.5.1", "python-dateutil==2.8.2", "pytz==2022.7.1", - "PyYAML==5.4.1", + "PyYAML==6.0.1", "requests==2.25.1", "yara-python==4.2.3", ], diff --git a/grr/proto/grr_response_proto/api/client.proto b/grr/proto/grr_response_proto/api/client.proto index c613930cf6..ed09a89e1a 100644 --- a/grr/proto/grr_response_proto/api/client.proto +++ b/grr/proto/grr_response_proto/api/client.proto @@ -23,7 +23,7 @@ message ApiClient { optional string urn = 1 [(sem_type) = { type: "ClientURN", description: "Client URN." }]; - optional bool fleetspeak_enabled = 20; + reserved 20; optional ClientInformation agent_info = 2; optional HardwareInfo hardware_info = 3; optional Uname os_info = 4; diff --git a/grr/proto/grr_response_proto/api/hunt.proto b/grr/proto/grr_response_proto/api/hunt.proto index a764d3e0ab..ad3cd1ce38 100644 --- a/grr/proto/grr_response_proto/api/hunt.proto +++ b/grr/proto/grr_response_proto/api/hunt.proto @@ -16,7 +16,7 @@ package grr; // Entities. // -// Next id: 36 +// Next id: 37 message ApiHunt { // Enum values here correspond to Hunt.State values. enum State { @@ -26,6 +26,18 @@ message ApiHunt { COMPLETED = 3; } + enum StateReason { + UNKNOWN = 0; + DEADLINE_REACHED = 1; + TOTAL_CLIENTS_EXCEEDED = 2; + TOTAL_CRASHES_EXCEEDED = 3; + TOTAL_NETWORK_EXCEEDED = 4; + AVG_RESULTS_EXCEEDED = 5; + AVG_NETWORK_EXCEEDED = 6; + AVG_CPU_EXCEEDED = 7; + TRIGGERED_BY_USER = 8; + } + enum HuntType { UNSET = 0; STANDARD = 1; @@ -43,6 +55,9 @@ message ApiHunt { optional State state = 4 [(sem_type) = { description: "Current hunt state." }]; + optional StateReason state_reason = 36 [(sem_type) = { + description: "Structured additional information on the hunt state." + }]; optional string state_comment = 33 [(sem_type) = { description: "Additional information on hunt state." }]; diff --git a/grr/proto/grr_response_proto/deprecated.proto b/grr/proto/grr_response_proto/deprecated.proto index cf117cf408..7e24f7ff10 100644 --- a/grr/proto/grr_response_proto/deprecated.proto +++ b/grr/proto/grr_response_proto/deprecated.proto @@ -468,3 +468,30 @@ message CheckFlowArgs { repeated uint64 max_findings = 4; repeated string restrict_checks = 5; } + +message CAEnrolerArgs { + optional Certificate csr = 1; +} + +message CollectSingleFileArgs { + optional string path = 1; + optional uint64 max_size_bytes = 2 [(sem_type) = { type: "ByteSize" }]; +} + +message CollectSingleFileResult { + optional StatEntry stat = 1; + optional Hash hash = 2; +} + +message CollectSingleFileProgress { + enum Status { + UNDEFINED = 0; + IN_PROGRESS = 1; + COLLECTED = 2; + NOT_FOUND = 3; + FAILED = 4; + } + optional Status status = 1; + optional CollectSingleFileResult result = 2; + optional string error_description = 3; +} diff --git a/grr/proto/grr_response_proto/export.proto b/grr/proto/grr_response_proto/export.proto index f5537ccd6d..5dea238a74 100644 --- a/grr/proto/grr_response_proto/export.proto +++ b/grr/proto/grr_response_proto/export.proto @@ -504,4 +504,4 @@ message ExportedExecuteResponse { optional bytes stdout = 5; optional bytes stderr = 6; optional uint64 time_used_us = 7; -} \ No newline at end of file +} diff --git a/grr/proto/grr_response_proto/flows.proto b/grr/proto/grr_response_proto/flows.proto index db7fc94961..4e5f92fa4c 100644 --- a/grr/proto/grr_response_proto/flows.proto +++ b/grr/proto/grr_response_proto/flows.proto @@ -192,12 +192,8 @@ message FlowRunnerArgs { label: HIDDEN, }]; - optional string base_session_id = 12 [(sem_type) = { - type: "RDFURN", - description: "The session id for the flow runner. " - "If not specified we make one.", - label: HIDDEN, - }]; + // DEPRECATED + // optional string base_session_id = 12; // DEPRECATED, unused. // optional uint64 start_time = 15; @@ -209,11 +205,8 @@ message FlowRunnerArgs { // label: ADVANCED // }, default="analysis/{p}/{u}-{t}"]; - optional string logs_collection_urn = 17 [(sem_type) = { - type: "RDFURN", - description: "The logs collection to log to for the flow.", - label: HIDDEN, - }]; + // DEPRECATED + // optional string logs_collection_urn = 17; optional bool write_intermediate_results = 18 [ (sem_type) = { @@ -355,11 +348,7 @@ message HuntRunnerArgs { ]; // DEPRECATED. - // optional string logs_collection_urn = 14 [(sem_type) = { - // type: "RDFURN", - // description: "The logs collection to log to for the hunt.", - // label: HIDDEN, - // }]; + // optional string logs_collection_urn = 14; optional bool add_foreman_rules = 16 [ default = true, @@ -510,13 +499,6 @@ message BareGrepSpec { // Various flows. -// Next field ID: 2 -message CAEnrolerArgs { - optional Certificate csr = 1 [(sem_type) = { - description: "A Certificate RDFValue with the CSR in it.", - }]; -} - // Next field ID: 2 message DeleteGRRTempFilesArgs { optional PathSpec pathspec = 1 [(sem_type) = { @@ -620,15 +602,7 @@ message ArtifactCollectorFlowArgs { }]; // Deprecated. - // TODO: Remove after 2021-04-01. - optional bool use_tsk = 2 [ - (sem_type) = { - description: "(Deprecated) Whether raw filesystem access should be used. " - "Use use_raw_filesystem_access insteaed." - label: HIDDEN, - }, - default = false - ]; + // optional bool use_tsk = 2; optional bool use_raw_filesystem_access = 15 [ (sem_type) = { @@ -745,15 +719,7 @@ message ArtifactFilesDownloaderFlowArgs { }]; // Deprecated. - // TODO: Remove after 2021-04-01. - optional bool use_tsk = 2 [ - (sem_type) = { - description: "(Deprecated) Whether raw filesystem access should be used. " - "Use use_raw_filesystem_access instead." - label: HIDDEN, - }, - default = false - ]; + // optional bool use_tsk = 2; optional bool use_raw_filesystem_access = 10 [ (sem_type) = { @@ -825,26 +791,6 @@ message FingerprintFileResult { }]; } -// Next field ID: 2 -message SophosCollectorArgs { - optional PathSpec.PathType pathtype = 1 [ - (sem_type) = { - description: "The requested path type.", - }, - default = OS - ]; -} - -// Next field ID: 2 -message MACTimesArgs { - optional string path = 1 [ - (sem_type) = { - description: "An AFF path (relative to the client area of the VFS).", - }, - default = "/fs/" - ]; -} - // Next field ID: 2 message FileCollectorArgs { repeated FindSpec findspecs = 1 [(sem_type) = { @@ -1267,22 +1213,6 @@ message ChromePluginsArgs { ]; } -// Next field ID: 3 -message UpdateVFSFileArgs { - optional string vfs_file_urn = 1 [(sem_type) = { - type: "RDFURN", - description: "VFSFile urn", - }]; - - optional string attribute = 2 [ - (sem_type) = { - description: "Attribute to update, given as the attribute name " - "on the schema class.", - }, - default = "CONTAINS" - ]; -} - // Next field ID: 6 message MultiGetFileArgs { repeated PathSpec pathspecs = 2 [(sem_type) = { @@ -1408,25 +1338,6 @@ message ListProcessesArgs { repeated uint32 pids = 4; } -// Next field ID: 3 -message ListVADBinariesArgs { - optional string filename_regex = 1 [ - (sem_type) = { - type: "RegularExpression", - friendly_name: "Filename Regex", - description: "Regex used to filter the list of binaries.", - }, - default = "." - ]; - - optional bool fetch_binaries = 2 [ - (sem_type) = { - friendly_name: "Fetch Binaries", - }, - default = false - ]; -} - // Next field ID: 7 message FileFinderModificationTimeCondition { optional uint64 min_last_modified_time = 5 [ @@ -1889,29 +1800,6 @@ message FileFinderResult { }]; } -message CollectSingleFileArgs { - optional string path = 1; - optional uint64 max_size_bytes = 2 [(sem_type) = { type: "ByteSize" }]; -} - -message CollectSingleFileResult { - optional StatEntry stat = 1; - optional Hash hash = 2; -} - -message CollectSingleFileProgress { - enum Status { - UNDEFINED = 0; - IN_PROGRESS = 1; - COLLECTED = 2; - NOT_FOUND = 3; - FAILED = 4; - } - optional Status status = 1; - optional CollectSingleFileResult result = 2; - optional string error_description = 3; -} - message CollectFilesByKnownPathArgs { // Path files to be collected. repeated string paths = 1; @@ -2013,6 +1901,44 @@ message CollectMultipleFilesProgress { optional uint64 num_failed = 5; } +message StatMultipleFilesArgs { + repeated string path_expressions = 1 [(sem_type) = { + type: "GlobExpression", + description: "Stats from files matching any of these expressions will be collected", + }]; + + optional FileFinderModificationTimeCondition modification_time = 2; + optional FileFinderAccessTimeCondition access_time = 3; + optional FileFinderInodeChangeTimeCondition inode_change_time = 4; + optional FileFinderSizeCondition size = 5; + optional FileFinderExtFlagsCondition ext_flags = 6; + optional FileFinderContentsRegexMatchCondition contents_regex_match = 7; + optional FileFinderContentsLiteralMatchCondition contents_literal_match = 8; +} + +message HashMultipleFilesArgs { + repeated string path_expressions = 1 [(sem_type) = { + type: "GlobExpression", + description: "Hashes from files matching any of these expressions will be collected", + }]; + + optional FileFinderModificationTimeCondition modification_time = 2; + optional FileFinderAccessTimeCondition access_time = 3; + optional FileFinderInodeChangeTimeCondition inode_change_time = 4; + optional FileFinderSizeCondition size = 5; + optional FileFinderExtFlagsCondition ext_flags = 6; + optional FileFinderContentsRegexMatchCondition contents_regex_match = 7; + optional FileFinderContentsLiteralMatchCondition contents_literal_match = 8; +} + +message HashMultipleFilesProgress { + optional uint64 num_found = 1; + optional uint64 num_in_progress = 2; + optional uint64 num_raw_fs_access_retries = 3; + optional uint64 num_hashed = 4; + optional uint64 num_failed = 5; +} + // Next field ID: 4 message FileReference { optional string client_id = 1 [(sem_type) = { diff --git a/grr/proto/grr_response_proto/hunts.proto b/grr/proto/grr_response_proto/hunts.proto index 60874ca745..a8d716e92c 100644 --- a/grr/proto/grr_response_proto/hunts.proto +++ b/grr/proto/grr_response_proto/hunts.proto @@ -41,7 +41,7 @@ message HuntArguments { optional HuntArgumentsVariable variable = 3; } -// Next id: 36 +// Next id: 37 message Hunt { optional string hunt_id = 1; optional string description = 2; @@ -123,6 +123,18 @@ message Hunt { COMPLETED = 4; } optional HuntState hunt_state = 18; + enum HuntStateReason { + UNKNOWN = 0; + DEADLINE_REACHED = 1; + TOTAL_CLIENTS_EXCEEDED = 2; + TOTAL_CRASHES_EXCEEDED = 3; + TOTAL_NETWORK_EXCEEDED = 4; + AVG_RESULTS_EXCEEDED = 5; + AVG_NETWORK_EXCEEDED = 6; + AVG_CPU_EXCEEDED = 7; + TRIGGERED_BY_USER = 8; + } + optional HuntStateReason hunt_state_reason = 36; optional string hunt_state_comment = 19; optional uint64 last_update_time = 20 [(sem_type) = { diff --git a/grr/proto/grr_response_proto/objects.proto b/grr/proto/grr_response_proto/objects.proto index 319c27cdcd..84f29dbe42 100644 --- a/grr/proto/grr_response_proto/objects.proto +++ b/grr/proto/grr_response_proto/objects.proto @@ -92,7 +92,7 @@ message ClientMetadata { optional bytes certificate = 1 [(sem_type) = { type: "RDFX509Cert", }]; - optional bool fleetspeak_enabled = 2; + reserved 2; optional uint64 ping = 3 [(sem_type) = { type: "RDFDatetime", description: "The last time the server heard from this client.", @@ -122,12 +122,7 @@ message ClientMetadata { }]; optional FleetspeakValidationInfo last_fleetspeak_validation_info = 10; - // Whether the client supports the RRG protocol. - // - // In reality, this field indicates whether the client has the RRG agent - // installed and running. This makes it possible to route certain action - // requests to the new agent. - optional bool rrg_support = 11; + reserved 11; } message ClientFullInfo { diff --git a/grr/proto/grr_response_proto/output_plugin.proto b/grr/proto/grr_response_proto/output_plugin.proto index 703ca4c233..4515bc7d1f 100644 --- a/grr/proto/grr_response_proto/output_plugin.proto +++ b/grr/proto/grr_response_proto/output_plugin.proto @@ -14,14 +14,7 @@ message OutputPluginDescriptor { optional string plugin_name = 1 [(sem_type) = { description: "The name of the output plugin." }]; - // TODO: Rename this field and add DEPRECATED prefix. - // DEPRECATED. Please use `args` instead. - optional bytes plugin_args = 2 [(sem_type) = { - dynamic_type: "GetPluginArgsClass", - description: "DEPRECATED. Please use `args` instead. The parameters for " - "this plugin. Must be an instance of the named plugin's " - "args_type." - }]; + optional bytes DEPRECATED_plugin_args = 2; optional google.protobuf.Any args = 3 [(sem_type) = { dynamic_type: "GetPluginArgsClass", diff --git a/grr/server/grr_response_server/artifact_registry.py b/grr/server/grr_response_server/artifact_registry.py index 25ef7e97d2..9847722070 100644 --- a/grr/server/grr_response_server/artifact_registry.py +++ b/grr/server/grr_response_server/artifact_registry.py @@ -10,7 +10,6 @@ from grr_response_core import config from grr_response_core.lib import artifact_utils -from grr_response_core.lib import objectfilter from grr_response_core.lib import parsers from grr_response_core.lib import type_info from grr_response_core.lib import utils @@ -543,16 +542,6 @@ def ValidateSyntax(rdf_artifact): detail = "invalid `supported_os` ('%s' not in %s)" % (supp_os, valid_os) raise rdf_artifacts.ArtifactSyntaxError(rdf_artifact, detail) - for condition in rdf_artifact.conditions: - # FIXME(hanuszczak): It does not look like the code below can throw - # `ConditionException`. Do we really need it then? - try: - of = objectfilter.Parser(condition).Parse() - of.Compile(objectfilter.BaseFilterImplementation) - except rdf_artifacts.ConditionError as e: - detail = "invalid condition '%s'" % condition - raise rdf_artifacts.ArtifactSyntaxError(rdf_artifact, detail, e) - # Anything listed in provides must be defined in the KnowledgeBase valid_provides = rdf_client.KnowledgeBase().GetKbFieldNames() for kb_var in rdf_artifact.provides: diff --git a/grr/server/grr_response_server/bigquery.py b/grr/server/grr_response_server/bigquery.py index 11a269651a..cc2b223cbc 100644 --- a/grr/server/grr_response_server/bigquery.py +++ b/grr/server/grr_response_server/bigquery.py @@ -2,7 +2,6 @@ """Library for interacting with Google BigQuery service.""" import json import logging -import time from googleapiclient import discovery from googleapiclient import errors @@ -10,7 +9,7 @@ import httplib2 from grr_response_core import config -from grr_response_core.lib import rdfvalue +from grr_response_core.lib.util import retry # pylint: disable=g-import-not-at-top @@ -117,51 +116,6 @@ def IsErrorRetryable(self, e): """ return e.resp.status in config.CONFIG["BigQuery.retry_status_codes"] - def RetryUpload(self, job, job_id, error): - """Retry the BigQuery upload job. - - Using the same job id protects us from duplicating data on the server. If we - fail all of our retries we raise. - - Args: - job: BigQuery job object - job_id: ID string for this upload job - error: errors.HttpError object from the first error - - Returns: - API response object on success, None on failure - Raises: - BigQueryJobUploadError: if we can't get the bigquery job started after - retry_max_attempts - """ - if self.IsErrorRetryable(error): - retry_count = 0 - sleep_interval = config.CONFIG["BigQuery.retry_interval"] - while retry_count < config.CONFIG["BigQuery.retry_max_attempts"]: - - time.sleep(sleep_interval.ToFractional(rdfvalue.SECONDS)) - logging.info("Retrying job_id: %s", job_id) - retry_count += 1 - - try: - response = job.execute() - return response - except errors.HttpError as e: - if self.IsErrorRetryable(e): - sleep_interval *= config.CONFIG["BigQuery.retry_multiplier"] - logging.exception("Error with job: %s, will retry in %s", job_id, - sleep_interval) - else: - raise BigQueryJobUploadError( - "Can't retry error code %s. Giving up" - " on job: %s." % (e.resp.status, job_id)) - else: - raise BigQueryJobUploadError("Can't retry error code %s. Giving up on " - "job: %s." % (error.resp.status, job_id)) - - raise BigQueryJobUploadError( - "Giving up on job:%s after %s retries." % (job_id, retry_count)) - def InsertData(self, table_id, fd, schema, job_id): """Insert data into a bigquery table. @@ -205,14 +159,37 @@ def InsertData(self, table_id, fd, schema, job_id): fd.name, mimetype="application/octet-stream") job = self.service.jobs().insert( projectId=self.project_id, body=body, media_body=mediafile) + + first_try = True + + @retry.When( + errors.HttpError, + self.IsErrorRetryable, + opts=retry.Opts( + attempts=config.CONFIG["BigQuery.retry_max_attempts"], + init_delay=config.CONFIG["BigQuery.retry_interval"].AsTimedelta(), + backoff=config.CONFIG["BigQuery.retry_multiplier"], + ), + ) + def Execute() -> None: + nonlocal first_try + + try: + job.execute() + except errors.HttpError: + if first_try: + first_try = False + + if self.GetDataset(self.dataset_id): + logging.exception("Error with job: %s", job_id) + else: + # If this is our first export ever, we need to create the dataset. + logging.info("Attempting to create dataset: %s", self.dataset_id) + self.CreateDataset() + + raise + try: - response = job.execute() - return response - except errors.HttpError as e: - if self.GetDataset(self.dataset_id): - logging.exception("Error with job: %s", job_id) - else: - # If this is our first export ever, we need to create the dataset. - logging.info("Attempting to create dataset: %s", self.dataset_id) - self.CreateDataset() - return self.RetryUpload(job, job_id, e) + Execute() + except errors.HttpError as error: + raise BigQueryJobUploadError(f"Failed job '{job_id}'") from error diff --git a/grr/server/grr_response_server/bigquery_test.py b/grr/server/grr_response_server/bigquery_test.py index dd8c0a88bc..3c1cf06438 100644 --- a/grr/server/grr_response_server/bigquery_test.py +++ b/grr/server/grr_response_server/bigquery_test.py @@ -49,36 +49,34 @@ def testInsertData(self, mock_http, mock_build, mock_creds): job_id, insert.call_args_list[0][1]["body"]["jobReference"]["jobId"]) def testRetryUpload(self): - bq_client = bigquery.BigQueryClient() + service = mock.Mock() + + bq_client = bigquery.BigQueryClient(bq_service=service) resp = mock.Mock() resp.status = 503 - error = mock.Mock() - error.resp = resp job = mock.Mock() # Always raise errors.HttpError on job.execute() job.configure_mock( **{"execute.side_effect": errors.HttpError(resp, b"nocontent")}) job_id = "hunts_HFFE1D044_Results_1446056474" + jobs = mock.Mock() + jobs.insert.return_value = job + + service.jobs.return_value = jobs + with temp.AutoTempFilePath() as filepath: with io.open(filepath, "w", encoding="utf-8") as filedesc: filedesc.write("{data}") - with mock.patch.object(time, "sleep") as mock_sleep: - with self.assertRaises(bigquery.BigQueryJobUploadError): - bq_client.RetryUpload(job, job_id, error) + with io.open(filepath, "rb") as filedesc: + with mock.patch.object(time, "sleep") as _: + with self.assertRaises(bigquery.BigQueryJobUploadError): + bq_client.InsertData("ExportedFile", filedesc, {}, job_id) - # Make sure retry sleeps are correct. max_calls = config.CONFIG["BigQuery.retry_max_attempts"] - retry_interval = config.CONFIG["BigQuery.retry_interval"] - multiplier = config.CONFIG["BigQuery.retry_multiplier"] - self.assertEqual(job.execute.call_count, max_calls) - mock_sleep.assert_has_calls([ - mock.call(retry_interval.ToFractional(rdfvalue.SECONDS)), - mock.call(retry_interval.ToFractional(rdfvalue.SECONDS) * multiplier) - ]) def main(argv): diff --git a/grr/server/grr_response_server/bin/fleetspeak_frontend.py b/grr/server/grr_response_server/bin/fleetspeak_frontend.py index ed74288da5..68b27c8739 100644 --- a/grr/server/grr_response_server/bin/fleetspeak_frontend.py +++ b/grr/server/grr_response_server/bin/fleetspeak_frontend.py @@ -5,16 +5,34 @@ import time from absl import app +from absl import flags from grr_response_core import config +from grr_response_core.config import server as config_server from grr_response_server import fleetspeak_connector from grr_response_server import server_startup from grr_response_server.bin import fleetspeak_frontend_server +_VERSION = flags.DEFINE_bool( + "version", + default=False, + allow_override=True, + help="Print the GRR Fleetspeak Frontend version and exit immediately.", +) + + def main(argv): del argv # Unused. + if _VERSION.value: + print( + "GRR Fleetspeak Frontend {}".format( + config_server.VERSION["packageversion"] + ) + ) + return + config.CONFIG.AddContext("FleetspeakFrontend Context") server_startup.Init() diff --git a/grr/server/grr_response_server/bin/fleetspeak_frontend_server.py b/grr/server/grr_response_server/bin/fleetspeak_frontend_server.py index c118db4a1b..93db7fa09f 100644 --- a/grr/server/grr_response_server/bin/fleetspeak_frontend_server.py +++ b/grr/server/grr_response_server/bin/fleetspeak_frontend_server.py @@ -1,7 +1,6 @@ #!/usr/bin/env python """This is the GRR frontend FS Server.""" import logging - from typing import Sequence import grpc @@ -22,6 +21,14 @@ "incoming_fleetspeak_messages", fields=[("status", str)] ) +FRONTEND_REQUEST_COUNT = metrics.Counter( + "frontend_request_count", fields=[("source", str)] +) + +FRONTEND_REQUEST_LATENCY = metrics.Event( + "frontend_request_latency", fields=[("source", str)] +) + # TODO: remove after the issue is fixed. CLIENT_ID_SKIP_LIST = frozenset( @@ -30,6 +37,13 @@ ) +MIN_DELAY_BETWEEN_METADATA_UPDATES = rdfvalue.Duration.From( + 30, rdfvalue.SECONDS +) + +WARN_IF_PROCESSING_LONGER_THAN = rdfvalue.Duration.From(30, rdfvalue.SECONDS) + + class GRRFSServer: """The GRR FS frontend server. @@ -39,17 +53,22 @@ class GRRFSServer: def __init__(self): self.frontend = frontend_lib.FrontEndServer( - certificate=config.CONFIG["Frontend.certificate"], - private_key=config.CONFIG["PrivateKeys.server_key"], max_queue_size=config.CONFIG["Frontend.max_queue_size"], message_expiry_time=config.CONFIG["Frontend.message_expiry_time"], max_retransmission_time=config .CONFIG["Frontend.max_retransmission_time"]) - @frontend_lib.FRONTEND_REQUEST_COUNT.Counted(fields=["fleetspeak"]) - @frontend_lib.FRONTEND_REQUEST_LATENCY.Timed(fields=["fleetspeak"]) + @FRONTEND_REQUEST_COUNT.Counted(fields=["fleetspeak"]) + @FRONTEND_REQUEST_LATENCY.Timed(fields=["fleetspeak"]) def Process(self, fs_msg: common_pb2.Message, context: grpc.ServicerContext): """Processes a single fleetspeak message.""" + request_start_time = rdfvalue.RDFDatetime.Now() + logged_actions = [] + + def _LogDelayed(msg: str) -> None: + elapsed = rdfvalue.RDFDatetime.Now() - request_start_time + logged_actions.append((elapsed, msg)) + fs_client_id = fs_msg.source.client_id grr_client_id = fleetspeak_utils.FleetspeakIDToGRRID(fs_client_id) @@ -65,34 +84,39 @@ def Process(self, fs_msg: common_pb2.Message, context: grpc.ServicerContext): validation_info = dict(fs_msg.validation_info.tags) try: - if fs_msg.source.service_name == "RRG": - rrg_support = True - else: - # `None` leaves the bit as it is stored in the database and `False` sets - # it to false. Since the server can receive messages from both agents, - # we don't want to set the `rrg_support` bit to `False` each time the - # Python agent sends something. - rrg_support = None - - client_is_new = self.frontend.EnrolFleetspeakClient(grr_client_id) - # TODO: We want to update metadata even if client is not new - # but is RRG-supported to set the `rrg_support` bit for older clients. In - # the future we should devise a smarter way of doing this but for now the - # amount of RRG-supported clients should be neglegible. - if not client_is_new or rrg_support: - data_store.REL_DB.WriteClientMetadata( - grr_client_id, - last_ping=rdfvalue.RDFDatetime.Now(), - fleetspeak_validation_info=validation_info, - rrg_support=rrg_support, - ) + _LogDelayed("Enrolling Fleetspeak client") + existing_client_mdata = self.frontend.EnrollFleetspeakClientIfNeeded( + grr_client_id + ) + _LogDelayed(f"Enrolled fleetspeak client: {existing_client_mdata}") + # Only update the client metadata if the client exists and the last + # update happened more than MIN_DELAY_BETWEEN_METADATA_UPDATES ago. + now = rdfvalue.RDFDatetime.Now() + if ( + existing_client_mdata is not None + and existing_client_mdata.ping is not None + ): + time_since_last_ping = now - existing_client_mdata.ping + if time_since_last_ping > MIN_DELAY_BETWEEN_METADATA_UPDATES: + _LogDelayed( + "Writing client metadata for existing client " + f"(time_since_last_ping={time_since_last_ping}" + ) + data_store.REL_DB.WriteClientMetadata( + grr_client_id, + last_ping=now, + fleetspeak_validation_info=validation_info, + ) + _LogDelayed("Written client metadata for existing client") if fs_msg.message_type == "GrrMessage": INCOMING_FLEETSPEAK_MESSAGES.Increment(fields=["PROCESS_GRR"]) grr_message = rdf_flows.GrrMessage.FromSerializedBytes( fs_msg.data.value) + _LogDelayed("Starting processing GRR message") self._ProcessGRRMessages(grr_client_id, [grr_message]) + _LogDelayed("Finished processing GRR message") elif fs_msg.message_type == "MessageList": INCOMING_FLEETSPEAK_MESSAGES.Increment( fields=["PROCESS_GRR_MESSAGE_LIST"] @@ -102,21 +126,27 @@ def Process(self, fs_msg: common_pb2.Message, context: grpc.ServicerContext): fs_msg.data.value) message_list = communicator.Communicator.DecompressMessageList( packed_messages) + _LogDelayed("Starting processing GRR message list") self._ProcessGRRMessages(grr_client_id, message_list.job) + _LogDelayed("Finished processing GRR message list") elif fs_msg.message_type == "rrg.Response": INCOMING_FLEETSPEAK_MESSAGES.Increment(fields=["PROCESS_RRG_RESPONSE"]) rrg_response = rrg_pb2.Response() rrg_response.ParseFromString(fs_msg.data.value) + _LogDelayed("Starting processing RRG response") self.frontend.ReceiveRRGResponse(grr_client_id, rrg_response) + _LogDelayed("Finished processing RRG response") elif fs_msg.message_type == "rrg.Parcel": INCOMING_FLEETSPEAK_MESSAGES.Increment(fields=["PROCESS_RRG_PARCEL"]) rrg_parcel = rrg_pb2.Parcel() rrg_parcel.ParseFromString(fs_msg.data.value) + _LogDelayed("Starting processing RRG parcel") self.frontend.ReceiveRRGParcel(grr_client_id, rrg_parcel) + _LogDelayed("Finished processing RRG parcel") else: INCOMING_FLEETSPEAK_MESSAGES.Increment(fields=["INVALID"]) @@ -126,6 +156,18 @@ def Process(self, fs_msg: common_pb2.Message, context: grpc.ServicerContext): except Exception: logging.exception("Exception processing message: %s", fs_msg) raise + finally: + total_elapsed = rdfvalue.RDFDatetime.Now() - request_start_time + if total_elapsed > WARN_IF_PROCESSING_LONGER_THAN: + logged_str = "\n".join( + "\t[{elapsed}]: {msg}".format(elapsed=elapsed, msg=msg) + for elapsed, msg in logged_actions + ) + logging.warning( + "Handling Fleetspeak frontend RPC took %s:\n%s", + total_elapsed, + logged_str, + ) def _ProcessGRRMessages( self, diff --git a/grr/server/grr_response_server/bin/fleetspeak_frontend_server_test.py b/grr/server/grr_response_server/bin/fleetspeak_frontend_server_test.py index 9086ae9a05..e089cbd657 100644 --- a/grr/server/grr_response_server/bin/fleetspeak_frontend_server_test.py +++ b/grr/server/grr_response_server/bin/fleetspeak_frontend_server_test.py @@ -27,10 +27,17 @@ class FleetspeakGRRFEServerTest(flow_test_lib.FlowTestsBaseclass): """Tests the Fleetspeak based GRRFEServer.""" def testReceiveMessages(self): + now = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(123) + fs_server = fleetspeak_frontend_server.GRRFSServer() client_id = "C.1234567890123456" flow_id = "12345678" - data_store.REL_DB.WriteClientMetadata(client_id, fleetspeak_enabled=True) + data_store.REL_DB.WriteClientMetadata( + client_id, + last_ping=now + - fleetspeak_frontend_server.MIN_DELAY_BETWEEN_METADATA_UPDATES + - rdfvalue.Duration("1s"), + ) rdf_flow = rdf_flow_objects.Flow( client_id=client_id, @@ -60,14 +67,13 @@ def testReceiveMessages(self): fs_message.validation_info.tags["foo"] = "bar" fs_messages.append(fs_message) - with test_lib.FakeTime(rdfvalue.RDFDatetime.FromSecondsSinceEpoch(123)): + with test_lib.FakeTime(now): for fs_message in fs_messages: fs_server.Process(fs_message, None) # Ensure the last-ping timestamp gets updated. client_data = data_store.REL_DB.MultiReadClientMetadata([client_id]) - self.assertEqual(client_data[client_id].ping, - rdfvalue.RDFDatetime.FromSecondsSinceEpoch(123)) + self.assertEqual(client_data[client_id].ping, now) self.assertEqual( client_data[client_id].last_fleetspeak_validation_info.ToStringDict(), {"foo": "bar"}) @@ -80,10 +86,16 @@ def testReceiveMessages(self): self.assertLen(flow_responses, 9) def testReceiveMessageList(self): + now = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(123) fs_server = fleetspeak_frontend_server.GRRFSServer() client_id = "C.1234567890123456" flow_id = "12345678" - data_store.REL_DB.WriteClientMetadata(client_id, fleetspeak_enabled=True) + data_store.REL_DB.WriteClientMetadata( + client_id, + last_ping=now + - fleetspeak_frontend_server.MIN_DELAY_BETWEEN_METADATA_UPDATES + - rdfvalue.Duration("1s"), + ) rdf_flow = rdf_flow_objects.Flow( client_id=client_id, @@ -115,13 +127,12 @@ def testReceiveMessageList(self): fs_message.data.Pack(packed_messages.AsPrimitiveProto()) fs_message.validation_info.tags["foo"] = "bar" - with test_lib.FakeTime(rdfvalue.RDFDatetime.FromSecondsSinceEpoch(123)): + with test_lib.FakeTime(now): fs_server.Process(fs_message, None) # Ensure the last-ping timestamp gets updated. client_data = data_store.REL_DB.MultiReadClientMetadata([client_id]) - self.assertEqual(client_data[client_id].ping, - rdfvalue.RDFDatetime.FromSecondsSinceEpoch(123)) + self.assertEqual(client_data[client_id].ping, now) self.assertEqual( client_data[client_id].last_fleetspeak_validation_info.ToStringDict(), {"foo": "bar"}) @@ -133,6 +144,86 @@ def testReceiveMessageList(self): self.assertEqual(stored_flow_request, flow_request) self.assertLen(flow_responses, 9) + def testMetadataDoesNotGetUpdatedIfPreviousUpdateIsTooRecent(self): + fs_server = fleetspeak_frontend_server.GRRFSServer() + client_id = "C.1234567890123456" + flow_id = "12345678" + now = rdfvalue.RDFDatetime.Now() + data_store.REL_DB.WriteClientMetadata(client_id, last_ping=now) + + rdf_flow = rdf_flow_objects.Flow( + client_id=client_id, + flow_id=flow_id, + create_time=rdfvalue.RDFDatetime.Now(), + ) + data_store.REL_DB.WriteFlowObject(rdf_flow) + flow_request = rdf_flow_objects.FlowRequest( + client_id=client_id, flow_id=flow_id, request_id=1 + ) + data_store.REL_DB.WriteFlowRequests([flow_request]) + session_id = "%s/%s" % (client_id, flow_id) + grr_message = rdf_flows.GrrMessage( + request_id=1, + response_id=1, + session_id=session_id, + payload=rdfvalue.RDFInteger(0), + ) + fs_client_id = fleetspeak_utils.GRRIDToFleetspeakID(client_id) + fs_message = fs_common_pb2.Message( + message_type="GrrMessage", + source=fs_common_pb2.Address( + client_id=fs_client_id, service_name=FS_SERVICE_NAME + ), + ) + fs_message.data.Pack(grr_message.AsPrimitiveProto()) + fs_server.Process(fs_message, None) + + # Ensure the last-ping timestamp doesn't get updated. + client_data = data_store.REL_DB.ReadClientMetadata(client_id) + self.assertEqual(client_data.ping, now) + + def testMetadataGetsUpdatedIfPreviousUpdateIsOldEnough(self): + fs_server = fleetspeak_frontend_server.GRRFSServer() + client_id = "C.1234567890123456" + flow_id = "12345678" + past = ( + rdfvalue.RDFDatetime.Now() + - fleetspeak_frontend_server.MIN_DELAY_BETWEEN_METADATA_UPDATES + - rdfvalue.Duration("1s") + ) + data_store.REL_DB.WriteClientMetadata(client_id, last_ping=past) + + rdf_flow = rdf_flow_objects.Flow( + client_id=client_id, + flow_id=flow_id, + create_time=rdfvalue.RDFDatetime.Now(), + ) + data_store.REL_DB.WriteFlowObject(rdf_flow) + flow_request = rdf_flow_objects.FlowRequest( + client_id=client_id, flow_id=flow_id, request_id=1 + ) + data_store.REL_DB.WriteFlowRequests([flow_request]) + session_id = "%s/%s" % (client_id, flow_id) + grr_message = rdf_flows.GrrMessage( + request_id=1, + response_id=1, + session_id=session_id, + payload=rdfvalue.RDFInteger(0), + ) + fs_client_id = fleetspeak_utils.GRRIDToFleetspeakID(client_id) + fs_message = fs_common_pb2.Message( + message_type="GrrMessage", + source=fs_common_pb2.Address( + client_id=fs_client_id, service_name=FS_SERVICE_NAME + ), + ) + fs_message.data.Pack(grr_message.AsPrimitiveProto()) + fs_server.Process(fs_message, None) + + # Ensure the last-ping timestamp does get updated. + client_data = data_store.REL_DB.ReadClientMetadata(client_id) + self.assertNotEqual(client_data.ping, past) + def testWriteLastPingForNewClients(self): fs_server = fleetspeak_frontend_server.GRRFSServer() client_id = "C.1234567890123456" @@ -180,7 +271,7 @@ class ListProcessesFleetspeakTest(flow_test_lib.FlowTestsBaseclass): def testProcessListingOnlyFleetspeak(self): """Test that the ListProcesses flow works with Fleetspeak.""" client_id = self.SetupClient(0) - data_store.REL_DB.WriteClientMetadata(client_id, fleetspeak_enabled=True) + data_store.REL_DB.WriteClientMetadata(client_id) client_mock = action_mocks.ListProcessesMock([ rdf_client.Process( diff --git a/grr/server/grr_response_server/bin/frontend.py b/grr/server/grr_response_server/bin/frontend.py deleted file mode 100644 index 055ec70f32..0000000000 --- a/grr/server/grr_response_server/bin/frontend.py +++ /dev/null @@ -1,322 +0,0 @@ -#!/usr/bin/env python -"""This is the GRR frontend HTTP Server.""" - -from http import server as http_server -import io -import ipaddress -import logging -import pdb -import socket -import socketserver -import threading -from urllib import parse as urlparse - -from absl import app -from absl import flags - -from grr_response_core import config -from grr_response_core.config import server as config_server -from grr_response_core.lib import rdfvalue -from grr_response_core.lib.rdfvalues import flows as rdf_flows -from grr_response_server import communicator -from grr_response_server import frontend_lib -from grr_response_server import server_logging -from grr_response_server import server_startup - -_VERSION = flags.DEFINE_bool( - "version", - default=False, - allow_override=True, - help="Print the GRR frontend version number and exit immediately.") - - -class GRRHTTPServerHandler(http_server.BaseHTTPRequestHandler): - """GRR HTTP handler for receiving client posts.""" - - statustext = { - 200: "200 OK", - 404: "404 Not Found", - 406: "406 Not Acceptable", - 500: "500 Internal Server Error" - } - - active_counter_lock = threading.Lock() - active_counter = 0 - - def _IncrementActiveCount(self): - with GRRHTTPServerHandler.active_counter_lock: - GRRHTTPServerHandler.active_counter += 1 - frontend_lib.FRONTEND_ACTIVE_COUNT.SetValue( - self.active_counter, fields=["http"]) - - def _DecrementActiveCount(self): - with GRRHTTPServerHandler.active_counter_lock: - GRRHTTPServerHandler.active_counter -= 1 - frontend_lib.FRONTEND_ACTIVE_COUNT.SetValue( - self.active_counter, fields=["http"]) - - def Send(self, - data, - status=200, - ctype="application/octet-stream", - additional_headers=None, - last_modified=0): - """Sends a response to the client.""" - if additional_headers: - additional_header_strings = [ - "%s: %s\r\n" % (name, val) - for name, val in additional_headers.items() - ] - else: - additional_header_strings = [] - - header = "" - header += "HTTP/1.0 %s\r\n" % self.statustext[status] - header += "Server: GRR Server\r\n" - header += "Content-type: %s\r\n" % ctype - header += "Content-Length: %d\r\n" % len(data) - header += "Last-Modified: %s\r\n" % self.date_time_string(last_modified) - header += "".join(additional_header_strings) - header += "\r\n" - - self.wfile.write(header.encode("utf-8")) - self.wfile.write(data) - - static_content_path = "/static/" - - def do_GET(self): # pylint: disable=g-bad-name - """Serve the server pem with GET requests.""" - self._IncrementActiveCount() - try: - if self.path.startswith("/server.pem"): - frontend_lib.FRONTEND_HTTP_REQUESTS.Increment(fields=["cert", "http"]) - self.ServerPem() - elif self.path.startswith(self.static_content_path): - frontend_lib.FRONTEND_HTTP_REQUESTS.Increment(fields=["static", "http"]) - self.ServeStatic(self.path[len(self.static_content_path):]) - finally: - self._DecrementActiveCount() - - def ServerPem(self): - self.Send(self.server.server_cert.AsPEM()) - - RECV_BLOCK_SIZE = 8192 - - def _GetPOSTData(self, length): - """Returns a specified number of bytes of the POST data.""" - # During our tests we have encountered some issue with the socket library - # that would stall for a long time when calling socket.recv(n) with a large - # n. rfile.read() passes the length down to socket.recv() so it's much - # faster to read the data in small 8k chunks. - input_data = io.BytesIO() - while length >= 0: - read_size = min(self.RECV_BLOCK_SIZE, length) - data = self.rfile.read(read_size) - if not data: - break - input_data.write(data) - length -= len(data) - return input_data.getvalue() - - def _GenerateChunk(self, length): - """Generates data for a single chunk.""" - - while 1: - to_read = min(length, self.RECV_BLOCK_SIZE) - if to_read == 0: - return - - data = self.rfile.read(to_read) - if not data: - return - - yield data - length -= len(data) - - def GenerateFileData(self): - """Generates the file data for a chunk encoded file.""" - # Handle chunked encoding: - # https://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.6.1 - while 1: - line = self.rfile.readline() - # We do not support chunked extensions, just ignore them. - chunk_size = int(line.split(";")[0], 16) - if chunk_size == 0: - break - - for chunk in self._GenerateChunk(chunk_size): - yield chunk - - # Chunk is followed by \r\n. - lf = self.rfile.read(2) - if lf != "\r\n": - raise IOError("Unable to parse chunk.") - - # Skip entity headers. - for header in self.rfile.readline(): - if not header: - break - - def do_POST(self): # pylint: disable=g-bad-name - """Process encrypted message bundles.""" - self._IncrementActiveCount() - try: - if self.path.startswith("/upload"): - frontend_lib.FRONTEND_HTTP_REQUESTS.Increment(fields=["upload", "http"]) - - logging.error("Requested no longer supported file upload through HTTP.") - self.Send(b"File upload though HTTP is no longer supported", status=404) - else: - frontend_lib.FRONTEND_HTTP_REQUESTS.Increment( - fields=["control", "http"]) - self.Control() - - except Exception as e: # pylint: disable=broad-except - if flags.FLAGS.pdb_post_mortem: - pdb.post_mortem() - - logging.exception("Had to respond with status 500.") - self.Send(("Error: %s" % e).encode("utf-8"), status=500) - finally: - self._DecrementActiveCount() - - @frontend_lib.FRONTEND_REQUEST_COUNT.Counted(fields=["http"]) - @frontend_lib.FRONTEND_REQUEST_LATENCY.Timed(fields=["http"]) - def Control(self): - """Handle POSTS.""" - # Get the api version - try: - api_version = int(urlparse.parse_qs(self.path.split("?")[1])["api"][0]) - except (ValueError, KeyError, IndexError): - # The oldest api version we support if not specified. - api_version = 3 - - try: - content_length = self.headers.get("content-length") - if not content_length: - raise IOError("No content-length header provided.") - - length = int(content_length) - - request_comms = rdf_flows.ClientCommunication.FromSerializedBytes( - self._GetPOSTData(length)) - - # If the client did not supply the version in the protobuf we use the get - # parameter. - if not request_comms.api_version: - request_comms.api_version = api_version - - # Reply using the same version we were requested with. - responses_comms = rdf_flows.ClientCommunication( - api_version=request_comms.api_version) - - address = self.client_address[0] - source_ip = ipaddress.ip_address(address) - - if source_ip.version == 6: - source_ip = source_ip.ipv4_mapped or source_ip - - request_comms.orig_request = rdf_flows.HttpRequest( - timestamp=rdfvalue.RDFDatetime.Now(), - raw_headers=str(self.headers), - source_ip=str(source_ip)) - - source, nr_messages = self.server.frontend.HandleMessageBundles( - request_comms, responses_comms) - - server_logging.LOGGER.LogHttpFrontendAccess( - request_comms.orig_request, source=source, message_count=nr_messages) - - self.Send(responses_comms.SerializeToBytes()) - - except communicator.UnknownClientCertError: - # "406 Not Acceptable: The server can only generate a response that is not - # accepted by the client". This is because we can not encrypt for the - # client appropriately. - self.Send(b"Enrollment required", status=406) - - -class GRRHTTPServer(socketserver.ThreadingMixIn, http_server.HTTPServer): - """The GRR HTTP frontend server.""" - - allow_reuse_address = True - request_queue_size = 500 - - address_family = socket.AF_INET6 - - def __init__(self, server_address, handler, frontend=None, **kwargs): - frontend_lib.FRONTEND_MAX_ACTIVE_COUNT.SetValue(self.request_queue_size) - - if frontend: - self.frontend = frontend - else: - self.frontend = frontend_lib.FrontEndServer( - certificate=config.CONFIG["Frontend.certificate"], - private_key=config.CONFIG["PrivateKeys.server_key"], - max_queue_size=config.CONFIG["Frontend.max_queue_size"], - message_expiry_time=config.CONFIG["Frontend.message_expiry_time"], - max_retransmission_time=config - .CONFIG["Frontend.max_retransmission_time"]) - self.server_cert = config.CONFIG["Frontend.certificate"] - - (address, _) = server_address - version = ipaddress.ip_address(address).version - if version == 4: - self.address_family = socket.AF_INET - elif version == 6: - self.address_family = socket.AF_INET6 - - logging.info("Will attempt to listen on %s", server_address) - http_server.HTTPServer.__init__(self, server_address, handler, **kwargs) - - def Shutdown(self): - self.shutdown() - - -def CreateServer(frontend=None): - """Start frontend http server.""" - max_port = config.CONFIG.Get("Frontend.port_max", - config.CONFIG["Frontend.bind_port"]) - - for port in range(config.CONFIG["Frontend.bind_port"], max_port + 1): - - server_address = (config.CONFIG["Frontend.bind_address"], port) - try: - httpd = GRRHTTPServer( - server_address, GRRHTTPServerHandler, frontend=frontend) - break - except socket.error as e: - if e.errno == socket.errno.EADDRINUSE and port < max_port: - logging.info("Port %s in use, trying %s", port, port + 1) - else: - raise - - sa = httpd.socket.getsockname() - logging.info("Serving HTTP on %s port %d ...", sa[0], sa[1]) - return httpd - - -def main(argv): - """Main.""" - del argv # Unused. - - if _VERSION.value: - print("GRR frontend {}".format(config_server.VERSION["packageversion"])) - return - - config.CONFIG.AddContext("HTTPServer Context") - - server_startup.Init() - - httpd = CreateServer() - - server_startup.DropPrivileges() - - try: - httpd.serve_forever() - except KeyboardInterrupt: - print("Caught keyboard interrupt, stopping") - - -if __name__ == "__main__": - app.run(main) diff --git a/grr/server/grr_response_server/bin/frontend_test.py b/grr/server/grr_response_server/bin/frontend_test.py deleted file mode 100644 index 8eace05487..0000000000 --- a/grr/server/grr_response_server/bin/frontend_test.py +++ /dev/null @@ -1,209 +0,0 @@ -#!/usr/bin/env python -"""Unittest for grr http server.""" - -import hashlib -import ipaddress -import os -import socket -import threading -import time - -from absl import app -import portpicker -import requests - -from grr_response_core.lib import utils -from grr_response_core.lib.rdfvalues import file_finder as rdf_file_finder -from grr_response_core.lib.rdfvalues import paths as rdf_paths -from grr_response_server import file_store -from grr_response_server.bin import frontend -from grr_response_server.databases import db -from grr_response_server.flows.general import file_finder -from grr.test_lib import action_mocks -from grr.test_lib import flow_test_lib -from grr.test_lib import test_lib -from grr.test_lib import worker_mocks - - -class GRRHTTPServerTest(test_lib.GRRBaseTest): - """Test the http server.""" - - @classmethod - def setUpClass(cls): - super(GRRHTTPServerTest, cls).setUpClass() - - # Bring up a local server for testing. - port = portpicker.pick_unused_port() - ip = utils.ResolveHostnameToIP("localhost", port) - cls.httpd = frontend.GRRHTTPServer((ip, port), - frontend.GRRHTTPServerHandler) - - if ipaddress.ip_address(ip).version == 6: - cls.address_family = socket.AF_INET6 - cls.base_url = "http://[%s]:%d/" % (ip, port) - else: - cls.address_family = socket.AF_INET - cls.base_url = "http://%s:%d/" % (ip, port) - - cls.httpd_thread = threading.Thread( - name="GRRHTTPServerTestThread", target=cls.httpd.serve_forever) - cls.httpd_thread.daemon = True - cls.httpd_thread.start() - - @classmethod - def tearDownClass(cls): - cls.httpd.Shutdown() - cls.httpd_thread.join() - - def setUp(self): - super().setUp() - self.client_id = self.SetupClient(0) - - def tearDown(self): - super().tearDown() - - # Wait until all pending http requests have been handled. - for _ in range(100): - if frontend.GRRHTTPServerHandler.active_counter == 0: - return - time.sleep(0.01) - self.fail("HTTP server thread did not shut down in time.") - - def testServerPem(self): - req = requests.get(self.base_url + "server.pem") - self.assertEqual(req.status_code, 200) - self.assertIn(b"BEGIN CERTIFICATE", req.content) - - def _RunClientFileFinder(self, - paths, - action, - network_bytes_limit=None, - client_id=None): - client_id = client_id or self.SetupClient(0) - with test_lib.ConfigOverrider({"Client.server_urls": [self.base_url]}): - session_id = flow_test_lib.TestFlowHelper( - file_finder.ClientFileFinder.__name__, - action_mocks.ClientFileFinderClientMock( - client_worker=worker_mocks.FakeClientWorker()), - client_id=client_id, - paths=paths, - pathtype=rdf_paths.PathSpec.PathType.OS, - action=action, - process_non_regular_files=True, - network_bytes_limit=network_bytes_limit, - creator=self.test_username) - - return session_id - - def testClientFileFinderUpload(self): - paths = [os.path.join(self.base_path, "{**,.}/*.plist")] - action = rdf_file_finder.FileFinderAction.Download() - - session_id = self._RunClientFileFinder(paths, action) - results = flow_test_lib.GetFlowResults(self.client_id, session_id) - self.assertLen(results, 5) - relpaths = [ - os.path.relpath(p.stat_entry.pathspec.path, self.base_path) - for p in results - ] - self.assertCountEqual(relpaths, [ - "History.plist", "History.xml.plist", "test.plist", - "parser_test/com.google.code.grr.plist", - "parser_test/InstallHistory.plist" - ]) - - for r in results: - data = open(r.stat_entry.pathspec.path, "rb").read() - - fd = file_store.OpenFile( - db.ClientPath.FromPathSpec(self.client_id, r.stat_entry.pathspec)) - self.assertEqual(fd.read(100), data[:100]) - - self.assertEqual(fd.hash_id.AsBytes(), hashlib.sha256(data).digest()) - - def testClientFileFinderUploadLimit(self): - paths = [os.path.join(self.base_path, "{**,.}/*.plist")] - action = rdf_file_finder.FileFinderAction.Download() - - # TODO(hanuszczak): Instead of catching arbitrary runtime errors, we should - # catch specific instance that was thrown. Unfortunately, all errors are - # intercepted in the `MockWorker` class and converted to runtime errors. - with self.assertRaisesRegex(RuntimeError, "exceeded network send limit"): - with test_lib.SuppressLogs(): - self._RunClientFileFinder(paths, action, network_bytes_limit=1500) - - def testClientFileFinderUploadBound(self): - paths = [os.path.join(self.base_path, "{**,.}/*.plist")] - action = rdf_file_finder.FileFinderAction.Download( - oversized_file_policy="DOWNLOAD_TRUNCATED", max_size=300) - - session_id = self._RunClientFileFinder(paths, action) - results = flow_test_lib.GetFlowResults(self.client_id, session_id) - self.assertLen(results, 5) - relpaths = [ - os.path.relpath(p.stat_entry.pathspec.path, self.base_path) - for p in results - ] - self.assertCountEqual(relpaths, [ - "History.plist", "History.xml.plist", "test.plist", - "parser_test/com.google.code.grr.plist", - "parser_test/InstallHistory.plist" - ]) - - def testClientFileFinderUploadSkip(self): - paths = [os.path.join(self.base_path, "{**,.}/*.plist")] - action = rdf_file_finder.FileFinderAction.Download( - oversized_file_policy="SKIP", max_size=300) - - session_id = self._RunClientFileFinder(paths, action) - results = flow_test_lib.GetFlowResults(self.client_id, session_id) - - skipped = [] - uploaded = [] - for result in results: - if result.HasField("transferred_file"): - uploaded.append(result) - else: - skipped.append(result) - - self.assertLen(uploaded, 2) - self.assertLen(skipped, 3) - - relpaths = [ - os.path.relpath(p.stat_entry.pathspec.path, self.base_path) - for p in uploaded - ] - self.assertCountEqual(relpaths, ["History.plist", "test.plist"]) - - def testClientFileFinderFilestoreIntegration(self): - paths = [os.path.join(self.base_path, "{**,.}/*.plist")] - action = rdf_file_finder.FileFinderAction.Download() - - client_ids = self.SetupClients(2) - session_ids = { - c: self._RunClientFileFinder(paths, action, client_id=c) - for c in client_ids - } - results_per_client = { - c: flow_test_lib.GetFlowResults(c, session_id) - for c, session_id in session_ids.items() - } - for results in results_per_client.values(): - self.assertLen(results, 5) - relpaths = [ - os.path.relpath(p.stat_entry.pathspec.path, self.base_path) - for p in results - ] - self.assertCountEqual(relpaths, [ - "History.plist", "History.xml.plist", "test.plist", - "parser_test/com.google.code.grr.plist", - "parser_test/InstallHistory.plist" - ]) - - -def main(args): - test_lib.main(args) - - -if __name__ == "__main__": - app.run(main) diff --git a/grr/server/grr_response_server/bin/grr_server.py b/grr/server/grr_response_server/bin/grr_server.py index cf395e520b..a4e4e44088 100644 --- a/grr/server/grr_response_server/bin/grr_server.py +++ b/grr/server/grr_response_server/bin/grr_server.py @@ -7,13 +7,11 @@ from absl import app from absl import flags -from grr_response_core import config from grr_response_core.config import server as config_server from grr_response_server import server_startup from grr_response_server.bin import fleetspeak_frontend from grr_response_server.bin import fleetspeak_server_wrapper -from grr_response_server.bin import frontend from grr_response_server.bin import grrafana from grr_response_server.bin import worker from grr_response_server.gui import admin_ui @@ -50,10 +48,7 @@ def main(argv): # Start as a frontend that clients communicate with. elif _COMPONENT.value.startswith("frontend"): server_startup.Init() - if config.CONFIG["Server.fleetspeak_enabled"]: - fleetspeak_frontend.main([argv]) - else: - frontend.main([argv]) + fleetspeak_frontend.main([argv]) # Start as an AdminUI. elif _COMPONENT.value.startswith("admin_ui"): diff --git a/grr/server/grr_response_server/client_index_test.py b/grr/server/grr_response_server/client_index_test.py index bd47b86b0e..789e9827ff 100644 --- a/grr/server/grr_response_server/client_index_test.py +++ b/grr/server/grr_response_server/client_index_test.py @@ -89,7 +89,7 @@ def testAddLookupClients(self): clients = self._SetupClients(2) for client_id, client in clients.items(): - data_store.REL_DB.WriteClientMetadata(client_id, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id) index.AddClient(client) # Check unique identifiers. @@ -135,8 +135,7 @@ def testAddTimestamp(self): # 1413807132 = Mon, 20 Oct 2014 12:12:12 GMT with test_lib.FakeTime(1413807132): for client_id, client in clients.items(): - data_store.REL_DB.WriteClientMetadata( - client_id, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id) index.AddClient(client) self.assertEqual( @@ -150,7 +149,7 @@ def testAddTimestamp(self): def testRemoveLabels(self): client_id = next(iter(self._SetupClients(1).keys())) data_store.REL_DB.WriteGRRUser("owner") - data_store.REL_DB.WriteClientMetadata(client_id, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id) data_store.REL_DB.AddClientLabels(client_id, "owner", ["testlabel_1", "testlabel_2"]) diff --git a/grr/server/grr_response_server/databases/db.py b/grr/server/grr_response_server/databases/db.py index daa65295ae..a28ecc416a 100644 --- a/grr/server/grr_response_server/databases/db.py +++ b/grr/server/grr_response_server/databases/db.py @@ -636,14 +636,12 @@ def WriteClientMetadata( self, client_id: str, certificate: Optional[rdf_crypto.RDFX509Cert] = None, - fleetspeak_enabled: Optional[bool] = None, first_seen: Optional[rdfvalue.RDFDatetime] = None, last_ping: Optional[rdfvalue.RDFDatetime] = None, last_clock: Optional[rdfvalue.RDFDatetime] = None, last_ip: Optional[rdf_client_network.NetworkAddress] = None, last_foreman: Optional[rdfvalue.RDFDatetime] = None, fleetspeak_validation_info: Optional[Mapping[str, str]] = None, - rrg_support: Optional[bool] = None, ) -> None: """Write metadata about the client. @@ -654,9 +652,6 @@ def WriteClientMetadata( client_id: A GRR client id string, e.g. "C.ea3b2b71840d6fa7". certificate: If set, should be an rdfvalues.crypto.RDFX509 protocol buffer. Normally only set during initial client record creation. - fleetspeak_enabled: A bool, indicating whether the client is connecting - through Fleetspeak. Normally only set during initial client record - creation. first_seen: An rdfvalue.Datetime, indicating the first time the client contacted the server. last_ping: An rdfvalue.Datetime, indicating the last time the client @@ -668,7 +663,6 @@ def WriteClientMetadata( last_foreman: An rdfvalue.Datetime, indicating the last time that the client sent a foreman message to the server. fleetspeak_validation_info: A dict with validation info from Fleetspeak. - rrg_support: Whether the agent supports the RRG protocol. """ @abc.abstractmethod @@ -810,17 +804,12 @@ def ReadAllClientIDs(self, def ReadClientLastPings(self, min_last_ping=None, max_last_ping=None, - fleetspeak_enabled=None, batch_size=CLIENT_IDS_BATCH_SIZE): """Yields dicts of last-ping timestamps for clients in the DB. Args: min_last_ping: The minimum timestamp to fetch from the DB. max_last_ping: The maximum timestamp to fetch from the DB. - fleetspeak_enabled: If set to True, only return data for - Fleetspeak-enabled clients. If set to False, only return ids for - non-Fleetspeak-enabled clients. If not set, return ids for both - Fleetspeak-enabled and non-Fleetspeak-enabled clients. batch_size: Integer, specifying the number of client pings to be queried at a time. @@ -889,6 +878,23 @@ def WriteClientRRGStartup( UnknownClientError: If the client is not known. """ + @abc.abstractmethod + def ReadClientRRGStartup( + self, + client_id: str, + ) -> Optional[rrg_startup_pb2.Startup]: + """Reads the latest RRG startup entry for the given client. + + Args: + client_id: An identifier of the client for which read the startup entry. + + Returns: + The latest startup entry if available and `None` if there are no such. + + Raises: + UnknownClientError: If the client is not known. + """ + @abc.abstractmethod def ReadClientStartupInfo(self, client_id: str) -> Optional[rdf_client.StartupInfo]: @@ -2551,15 +2557,18 @@ def WriteHuntObject(self, hunt_obj): """ @abc.abstractmethod - def UpdateHuntObject(self, - hunt_id, - duration=None, - client_rate=None, - client_limit=None, - hunt_state=None, - hunt_state_comment=None, - start_time=None, - num_clients_at_start_time=None): + def UpdateHuntObject( + self, + hunt_id, + duration=None, + client_rate=None, + client_limit=None, + hunt_state=None, + hunt_state_reason=None, + hunt_state_comment=None, + start_time=None, + num_clients_at_start_time=None, + ): """Updates the hunt object by applying the update function. Each keyword argument when set to None, means that that corresponding value @@ -2568,10 +2577,11 @@ def UpdateHuntObject(self, Args: hunt_id: Id of the hunt to be updated. duration: A maximum allowed running time duration of the flow. - client_rate: Number correpsonding to hunt's client rate. + client_rate: Number corresponding to hunt's client rate. client_limit: Number corresponding hunt's client limit. hunt_state: New Hunt.HuntState value. - hunt_state_comment: String correpsonding to a hunt state comment. + hunt_state_reason: New Hunt.HuntStateReason value. + hunt_state_comment: String corresponding to a hunt state comment. start_time: RDFDatetime corresponding to a start time of the hunt. num_clients_at_start_time: Integer corresponding to a number of clients at start time. @@ -3127,18 +3137,15 @@ def WriteClientMetadata( self, client_id: str, certificate: Optional[rdf_crypto.RDFX509Cert] = None, - fleetspeak_enabled: Optional[bool] = None, first_seen: Optional[rdfvalue.RDFDatetime] = None, last_ping: Optional[rdfvalue.RDFDatetime] = None, last_clock: Optional[rdfvalue.RDFDatetime] = None, last_ip: Optional[rdf_client_network.NetworkAddress] = None, last_foreman: Optional[rdfvalue.RDFDatetime] = None, fleetspeak_validation_info: Optional[Mapping[str, str]] = None, - rrg_support: Optional[bool] = None, ) -> None: precondition.ValidateClientId(client_id) precondition.AssertOptionalType(certificate, rdf_crypto.RDFX509Cert) - precondition.AssertOptionalType(fleetspeak_enabled, bool) precondition.AssertOptionalType(first_seen, rdfvalue.RDFDatetime) precondition.AssertOptionalType(last_ping, rdfvalue.RDFDatetime) precondition.AssertOptionalType(last_clock, rdfvalue.RDFDatetime) @@ -3151,14 +3158,12 @@ def WriteClientMetadata( return self.delegate.WriteClientMetadata( client_id, certificate=certificate, - fleetspeak_enabled=fleetspeak_enabled, first_seen=first_seen, last_ping=last_ping, last_clock=last_clock, last_ip=last_ip, last_foreman=last_foreman, fleetspeak_validation_info=fleetspeak_validation_info, - rrg_support=rrg_support, ) def DeleteClient(self, client_id): @@ -3191,11 +3196,9 @@ def MultiReadClientFullInfo(self, client_ids, min_last_ping=None): def ReadClientLastPings(self, min_last_ping=None, max_last_ping=None, - fleetspeak_enabled=None, batch_size=CLIENT_IDS_BATCH_SIZE): precondition.AssertOptionalType(min_last_ping, rdfvalue.RDFDatetime) precondition.AssertOptionalType(max_last_ping, rdfvalue.RDFDatetime) - precondition.AssertOptionalType(fleetspeak_enabled, bool) precondition.AssertType(batch_size, int) if batch_size < 1: @@ -3206,7 +3209,6 @@ def ReadClientLastPings(self, return self.delegate.ReadClientLastPings( min_last_ping=min_last_ping, max_last_ping=max_last_ping, - fleetspeak_enabled=fleetspeak_enabled, batch_size=batch_size) def WriteClientSnapshotHistory(self, clients): @@ -3248,6 +3250,12 @@ def WriteClientRRGStartup( ) -> None: return self.delegate.WriteClientRRGStartup(client_id, startup) + def ReadClientRRGStartup( + self, + client_id: str, + ) -> Optional[rrg_startup_pb2.Startup]: + return self.delegate.ReadClientRRGStartup(client_id) + def ReadClientStartupInfo(self, client_id: str) -> Optional[rdf_client.StartupInfo]: precondition.ValidateClientId(client_id) @@ -4229,15 +4237,18 @@ def WriteHuntObject(self, hunt_obj): self.delegate.WriteHuntObject(hunt_obj) - def UpdateHuntObject(self, - hunt_id, - duration=None, - client_rate=None, - client_limit=None, - hunt_state=None, - hunt_state_comment=None, - start_time=None, - num_clients_at_start_time=None): + def UpdateHuntObject( + self, + hunt_id, + duration=None, + client_rate=None, + client_limit=None, + hunt_state=None, + hunt_state_reason=None, + hunt_state_comment=None, + start_time=None, + num_clients_at_start_time=None, + ): """Updates the hunt object by applying the update function.""" _ValidateHuntId(hunt_id) precondition.AssertOptionalType(duration, rdfvalue.Duration) @@ -4245,6 +4256,8 @@ def UpdateHuntObject(self, precondition.AssertOptionalType(client_limit, int) if hunt_state is not None: _ValidateEnumType(hunt_state, rdf_hunt_objects.Hunt.HuntState) + if hunt_state_reason is not None: + _ValidateEnumType(hunt_state, rdf_hunt_objects.Hunt.HuntStateReason) precondition.AssertOptionalType(hunt_state_comment, str) precondition.AssertOptionalType(start_time, rdfvalue.RDFDatetime) precondition.AssertOptionalType(num_clients_at_start_time, int) @@ -4255,9 +4268,11 @@ def UpdateHuntObject(self, client_rate=client_rate, client_limit=client_limit, hunt_state=hunt_state, + hunt_state_reason=hunt_state_reason, hunt_state_comment=hunt_state_comment, start_time=start_time, - num_clients_at_start_time=num_clients_at_start_time) + num_clients_at_start_time=num_clients_at_start_time, + ) def ReadHuntOutputPluginsStates(self, hunt_id): _ValidateHuntId(hunt_id) diff --git a/grr/server/grr_response_server/databases/db_clients_test.py b/grr/server/grr_response_server/databases/db_clients_test.py index c9444797de..71757a513d 100644 --- a/grr/server/grr_response_server/databases/db_clients_test.py +++ b/grr/server/grr_response_server/databases/db_clients_test.py @@ -125,7 +125,7 @@ def testClientMetadataInitialWrite(self): client_id_1 = "C.fc413187fefa1dcf" # Typical initial FS enabled write - d.WriteClientMetadata(client_id_1, fleetspeak_enabled=True) + d.WriteClientMetadata(client_id_1) client_id_2 = "C.00413187fefa1dcf" # Typical initial non-FS write @@ -133,18 +133,16 @@ def testClientMetadataInitialWrite(self): client_id_2, certificate=CERT, first_seen=rdfvalue.RDFDatetime(100000000), - fleetspeak_enabled=False) + ) res = d.MultiReadClientMetadata([client_id_1, client_id_2]) self.assertLen(res, 2) m1 = res[client_id_1] self.assertIsInstance(m1, rdf_objects.ClientMetadata) - self.assertTrue(m1.fleetspeak_enabled) m2 = res[client_id_2] self.assertIsInstance(m2, rdf_objects.ClientMetadata) - self.assertFalse(m2.fleetspeak_enabled) self.assertEqual(m2.certificate, CERT) self.assertEqual(m2.first_seen, rdfvalue.RDFDatetime(100000000)) @@ -157,7 +155,7 @@ def testClientMetadataSubsecond(self): last_clock=rdfvalue.RDFDatetime(100000011), last_foreman=rdfvalue.RDFDatetime(100000021), last_ping=rdfvalue.RDFDatetime(100000031), - fleetspeak_enabled=False) + ) res = self.db.MultiReadClientMetadata([client_id]) self.assertLen(res, 1) m1 = res[client_id] @@ -174,7 +172,6 @@ def testClientMetadataPing(self): # Typical update on client ping. d.WriteClientMetadata( client_id, - fleetspeak_enabled=True, last_ping=rdfvalue.RDFDatetime(200000000000), last_clock=rdfvalue.RDFDatetime(210000000000), last_ip=rdf_client_network.NetworkAddress( @@ -185,7 +182,6 @@ def testClientMetadataPing(self): self.assertLen(res, 1) m1 = res[client_id] self.assertIsInstance(m1, rdf_objects.ClientMetadata) - self.assertTrue(m1.fleetspeak_enabled) self.assertEqual(m1.ping, rdfvalue.RDFDatetime(200000000000)) self.assertEqual(m1.clock, rdfvalue.RDFDatetime(210000000000)) self.assertEqual( @@ -197,29 +193,7 @@ def testClientMetadataValidatesIP(self): d = self.db client_id = "C.fc413187fefa1dcf" with self.assertRaises(TypeError): - d.WriteClientMetadata( - client_id, fleetspeak_enabled=True, last_ip="127.0.0.1") - - def testClientMetadataRrgSupportUnset(self): - client_id = "C.1234567890abcdef" - self.db.WriteClientMetadata(client_id, fleetspeak_enabled=True) - - metadata = self.db.ReadClientMetadata(client_id) - self.assertFalse(metadata.rrg_support) - - def testClientMetadataRrgSupportFalse(self): - client_id = "C.1234567890abcdef" - self.db.WriteClientMetadata(client_id, rrg_support=False) - - metadata = self.db.ReadClientMetadata(client_id) - self.assertFalse(metadata.rrg_support) - - def testClientMetadataRrgSupportTrue(self): - client_id = "C.1234567890abcdef" - self.db.WriteClientMetadata(client_id, rrg_support=True) - - metadata = self.db.ReadClientMetadata(client_id) - self.assertTrue(metadata.rrg_support) + d.WriteClientMetadata(client_id, last_ip="127.0.0.1") def testReadAllClientIDsEmpty(self): result = list(self.db.ReadAllClientIDs()) @@ -258,7 +232,7 @@ def testReadAllClientIDsEvenlyDivisibleByBatchSize(self): [client_a_id, client_b_id, client_c_id, client_d_id]) def testReadAllClientIDsFilterLastPing(self): - self.db.WriteClientMetadata("C.0000000000000001", fleetspeak_enabled=True) + self.db.WriteClientMetadata("C.0000000000000001") self.db.WriteClientMetadata( "C.0000000000000002", last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(2)) @@ -337,64 +311,65 @@ def testReadClientLastPings_NoFilter(self): client_id10: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(5), }]) - def testReadClientLastPings_AllFiltersFleetspeak(self): - client_ids = self._WriteClientLastPingData() - client_id6 = client_ids[5] - client_id8 = client_ids[7] - - actual_data = self.db.ReadClientLastPings( - min_last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3), - max_last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(4), - fleetspeak_enabled=True) - expected_data = [{ - client_id6: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3), - client_id8: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(4), - }] - self.assertEqual(list(actual_data), expected_data) - - def testReadClientLastPings_AllFiltersNoFleetspeak(self): + def testReadClientLastPings_AllFilters(self): client_ids = self._WriteClientLastPingData() client_id5 = client_ids[4] + client_id6 = client_ids[5] client_id7 = client_ids[6] + client_id8 = client_ids[7] actual_data = self.db.ReadClientLastPings( min_last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3), max_last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(4), - fleetspeak_enabled=False) + ) expected_data = [{ client_id5: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3), + client_id6: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3), client_id7: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(4), + client_id8: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(4), }] self.assertEqual(list(actual_data), expected_data) - def testReadClientLastPings_MinPingFleetspeakFilters(self): + def testReadClientLastPings_MinPingFilter(self): client_ids = self._WriteClientLastPingData() client_id5 = client_ids[4] + client_id6 = client_ids[5] client_id7 = client_ids[6] + client_id8 = client_ids[7] client_id9 = client_ids[8] + client_id10 = client_ids[9] actual_data = self.db.ReadClientLastPings( - min_last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3), - fleetspeak_enabled=False) + min_last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3) + ) expected_data = [{ client_id5: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3), + client_id6: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3), client_id7: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(4), + client_id8: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(4), client_id9: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(5), + client_id10: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(5), }] self.assertEqual(list(actual_data), expected_data) - def testReadClientLastPings_MaxPingFleetspeakFilters(self): + def testReadClientLastPings_MaxPingFilter(self): client_ids = self._WriteClientLastPingData() + client_id1 = client_ids[0] client_id2 = client_ids[1] + client_id3 = client_ids[2] client_id4 = client_ids[3] + client_id5 = client_ids[4] client_id6 = client_ids[5] actual_data = self.db.ReadClientLastPings( - max_last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3), - fleetspeak_enabled=True) + max_last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3) + ) expected_data = [{ + client_id1: None, client_id2: None, + client_id3: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(2), client_id4: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(2), + client_id5: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3), client_id6: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3), }] self.assertEqual(list(actual_data), expected_data) @@ -405,32 +380,28 @@ def _WriteClientLastPingData(self): (client_id1, client_id2, client_id3, client_id4, client_id5, client_id6, client_id7, client_id8, client_id9, client_id10) = client_ids - self.db.WriteClientMetadata(client_id1, fleetspeak_enabled=False) - self.db.WriteClientMetadata(client_id2, fleetspeak_enabled=True) + self.db.WriteClientMetadata(client_id1) + self.db.WriteClientMetadata(client_id2) self.db.WriteClientMetadata( client_id3, last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(2)) self.db.WriteClientMetadata( - client_id4, - last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(2), - fleetspeak_enabled=True) + client_id4, last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(2) + ) self.db.WriteClientMetadata( client_id5, last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3)) self.db.WriteClientMetadata( - client_id6, - last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3), - fleetspeak_enabled=True) + client_id6, last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3) + ) self.db.WriteClientMetadata( client_id7, last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(4)) self.db.WriteClientMetadata( - client_id8, - last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(4), - fleetspeak_enabled=True) + client_id8, last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(4) + ) self.db.WriteClientMetadata( client_id9, last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(5)) self.db.WriteClientMetadata( - client_id10, - last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(5), - fleetspeak_enabled=True) + client_id10, last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(5) + ) return client_ids @@ -1378,6 +1349,62 @@ def testWriteClientRRGStartupMultipleClients(self): info_2 = self.db.ReadClientFullInfo(client_id_2) self.assertEqual(info_2.last_rrg_startup.AsPrimitiveProto(), startup_2) + def testReadClientRRGStartupUnknownClient(self): + with self.assertRaises(db.UnknownClientError): + self.db.ReadClientRRGStartup("C.0123456789ABCDEF") + + def testReadClientRRGStartupNone(self): + client_id = db_test_utils.InitializeClient(self.db) + + self.assertIsNone(self.db.ReadClientRRGStartup(client_id)) + + def testReadClientRRGStartupSingle(self): + client_id = db_test_utils.InitializeClient(self.db) + + startup = rrg_startup_pb2.Startup() + startup.metadata.version.major = 1 + startup.metadata.version.minor = 2 + startup.metadata.version.patch = 3 + self.db.WriteClientRRGStartup(client_id, startup) + + self.assertEqual(self.db.ReadClientRRGStartup(client_id), startup) + + def testReadClientRRGStartupMultipleStartups(self): + client_id = db_test_utils.InitializeClient(self.db) + + startup_1 = rrg_startup_pb2.Startup() + startup_1.metadata.version.major = 1 + startup_1.metadata.version.minor = 2 + startup_1.metadata.version.patch = 3 + self.db.WriteClientRRGStartup(client_id, startup_1) + + startup_2 = rrg_startup_pb2.Startup() + startup_2.metadata.version.major = 4 + startup_2.metadata.version.minor = 5 + startup_2.metadata.version.patch = 6 + self.db.WriteClientRRGStartup(client_id, startup_2) + + self.assertEqual(self.db.ReadClientRRGStartup(client_id), startup_2) + + def testReadClientRRGStartupMultipleClients(self): + client_id_1 = db_test_utils.InitializeClient(self.db) + client_id_2 = db_test_utils.InitializeClient(self.db) + + startup_1 = rrg_startup_pb2.Startup() + startup_1.metadata.version.major = 1 + startup_1.metadata.version.minor = 2 + startup_1.metadata.version.patch = 3 + self.db.WriteClientRRGStartup(client_id_1, startup_1) + + startup_2 = rrg_startup_pb2.Startup() + startup_2.metadata.version.major = 4 + startup_2.metadata.version.minor = 5 + startup_2.metadata.version.patch = 6 + self.db.WriteClientRRGStartup(client_id_2, startup_2) + + self.assertEqual(self.db.ReadClientRRGStartup(client_id_1), startup_1) + self.assertEqual(self.db.ReadClientRRGStartup(client_id_2), startup_2) + def testCrashHistory(self): d = self.db @@ -1579,7 +1606,7 @@ def testMultiReadClientsFullInfoSkipsMissingClients(self): present_client_id = "C.fc413187fefa1dcf" # Typical initial FS enabled write - d.WriteClientMetadata(present_client_id, fleetspeak_enabled=True) + d.WriteClientMetadata(present_client_id) missing_client_id = "C.00413187fefa1dcf" @@ -1591,7 +1618,7 @@ def testMultiReadClientsFullInfoNoSnapshot(self): d = self.db client_id = "C.fc413187fefa1dcf" - d.WriteClientMetadata(client_id, fleetspeak_enabled=True) + d.WriteClientMetadata(client_id) full_info = d.MultiReadClientFullInfo([client_id])[client_id] expected_snapshot = rdf_objects.ClientSnapshot(client_id=client_id) self.assertEqual(full_info.last_snapshot, expected_snapshot) @@ -1776,8 +1803,7 @@ def _WriteTestClientsWithData(self, labels_dict=None): for index in client_indices: client_id = "C.1%015x" % index - self.db.WriteClientMetadata( - client_id, last_ping=last_ping, fleetspeak_enabled=False) + self.db.WriteClientMetadata(client_id, last_ping=last_ping) self.db.WriteClientSnapshot( rdf_objects.ClientSnapshot( client_id=client_id, diff --git a/grr/server/grr_response_server/databases/db_cronjob_test.py b/grr/server/grr_response_server/databases/db_cronjob_test.py index b9acc07b55..c359ff7f68 100644 --- a/grr/server/grr_response_server/databases/db_cronjob_test.py +++ b/grr/server/grr_response_server/databases/db_cronjob_test.py @@ -5,7 +5,9 @@ from grr_response_core.lib import rdfvalue from grr_response_core.lib.util import random from grr_response_server.databases import db +from grr_response_server.databases import db_test_utils from grr_response_server.rdfvalues import cronjobs as rdf_cronjobs +from grr_response_server.rdfvalues import objects as rdf_objects from grr.test_lib import test_lib @@ -109,6 +111,28 @@ def testCronJobDeletion_UnknownJob(self): with self.assertRaises(db.UnknownCronJobError): self.db.DeleteCronJob("non-existent-id") + def testDeleteCronJobWithApprovalRequest(self): + creator = db_test_utils.InitializeUser(self.db) + approver = db_test_utils.InitializeUser(self.db) + cron_job_id = db_test_utils.InitializeCronJob(self.db) + + approval = rdf_objects.ApprovalRequest() + approval.approval_type = ( + rdf_objects.ApprovalRequest.ApprovalType.APPROVAL_TYPE_CRON_JOB + ) + approval.requestor_username = creator + approval.notified_users = [approver] + approval.subject_id = cron_job_id + approval.expiration_time = ( + rdfvalue.RDFDatetime.Now() + rdfvalue.Duration.From(1, rdfvalue.DAYS) + ) + approval_id = self.db.WriteApprovalRequest(approval) + + self.db.DeleteCronJob(cron_job_id) + + with self.assertRaises(db.UnknownApprovalRequestError): + self.db.ReadApprovalRequest(creator, approval_id) + def testCronJobEnabling(self): job = self._CreateCronJob() self.db.WriteCronJob(job) diff --git a/grr/server/grr_response_server/databases/db_flows_test.py b/grr/server/grr_response_server/databases/db_flows_test.py index ee52f54dd4..bc2b5928ca 100644 --- a/grr/server/grr_response_server/databases/db_flows_test.py +++ b/grr/server/grr_response_server/databases/db_flows_test.py @@ -222,7 +222,7 @@ def testFlowWriting(self): flow_id = u"1234ABCD" client_id = u"C.1234567890123456" - self.db.WriteClientMetadata(client_id, fleetspeak_enabled=False) + self.db.WriteClientMetadata(client_id) rdf_flow = rdf_flow_objects.Flow( client_id=client_id, @@ -251,7 +251,7 @@ def testFlowOverwrite(self): flow_id = u"1234ABCD" client_id = u"C.1234567890123456" - self.db.WriteClientMetadata(client_id, fleetspeak_enabled=False) + self.db.WriteClientMetadata(client_id) rdf_flow = rdf_flow_objects.Flow( client_id=client_id, flow_id=flow_id, next_request_to_process=4) @@ -279,7 +279,7 @@ def testFlowOverwriteFailsWithAllowUpdateFalse(self): flow_id = u"1234ABCD" client_id = u"C.1234567890123456" - self.db.WriteClientMetadata(client_id, fleetspeak_enabled=False) + self.db.WriteClientMetadata(client_id) rdf_flow = rdf_flow_objects.Flow( client_id=client_id, flow_id=flow_id, next_request_to_process=4) @@ -301,7 +301,7 @@ def testFlowTimestamp(self): client_id = "C.0123456789012345" flow_id = "0F00B430" - self.db.WriteClientMetadata(client_id, fleetspeak_enabled=False) + self.db.WriteClientMetadata(client_id) before_timestamp = self.db.Now() @@ -317,7 +317,7 @@ def testFlowTimestampWithMissingCreationTime(self): client_id = "C.0123456789012345" flow_id = "0F00B430" - self.db.WriteClientMetadata(client_id, fleetspeak_enabled=False) + self.db.WriteClientMetadata(client_id) before_timestamp = self.db.Now() @@ -334,7 +334,7 @@ def testFlowNameWithMissingNameInProtobuf(self): client_id = "C.0123456789012345" flow_id = "0F00B430" - self.db.WriteClientMetadata(client_id, fleetspeak_enabled=False) + self.db.WriteClientMetadata(client_id) flow_obj = rdf_flow_objects.Flow(client_id=client_id, flow_id=flow_id) flow_obj.flow_class_name = "Quux" @@ -350,7 +350,7 @@ def testFlowKeyMetadataUnchangable(self): client_id = "C.0123456789012345" flow_id = "0F00B430" - self.db.WriteClientMetadata(client_id, fleetspeak_enabled=False) + self.db.WriteClientMetadata(client_id) flow_obj = rdf_flow_objects.Flow(client_id=client_id, flow_id=flow_id) flow_obj.long_flow_id = f"{client_id}/{flow_id}" @@ -394,7 +394,7 @@ def testFlowNameUnchangable(self): client_id = "C.0123456789012345" flow_id = "0F00B430" - self.db.WriteClientMetadata(client_id, fleetspeak_enabled=False) + self.db.WriteClientMetadata(client_id) flow_obj = rdf_flow_objects.Flow(client_id=client_id, flow_id=flow_id) flow_obj.flow_class_name = "Quux" @@ -410,7 +410,7 @@ def testFlowCreatorUnchangable(self): client_id = "C.0123456789012345" flow_id = "0F00B430" - self.db.WriteClientMetadata(client_id, fleetspeak_enabled=False) + self.db.WriteClientMetadata(client_id) flow_obj = rdf_flow_objects.Flow(client_id=client_id, flow_id=flow_id) flow_obj.creator = "norf" @@ -426,7 +426,7 @@ def testFlowCreatorUnsetInProtobuf(self): client_id = "C.0123456789012345" flow_id = "0F00B430" - self.db.WriteClientMetadata(client_id, fleetspeak_enabled=False) + self.db.WriteClientMetadata(client_id) flow_obj = rdf_flow_objects.Flow(client_id=client_id, flow_id=flow_id) flow_obj.creator = "norf" @@ -442,8 +442,8 @@ def testReadAllFlowObjects(self): client_id_1 = "C.1111111111111111" client_id_2 = "C.2222222222222222" - self.db.WriteClientMetadata(client_id_1, fleetspeak_enabled=False) - self.db.WriteClientMetadata(client_id_2, fleetspeak_enabled=False) + self.db.WriteClientMetadata(client_id_1) + self.db.WriteClientMetadata(client_id_2) # Write a flow and a child flow for client 1. flow1 = rdf_flow_objects.Flow(client_id=client_id_1, flow_id="000A0001") @@ -462,7 +462,7 @@ def testReadAllFlowObjects(self): def testReadAllFlowObjectsWithMinCreateTime(self): client_id_1 = "C.1111111111111111" - self.db.WriteClientMetadata(client_id_1, fleetspeak_enabled=False) + self.db.WriteClientMetadata(client_id_1) self.db.WriteFlowObject( rdf_flow_objects.Flow(client_id=client_id_1, flow_id="0000001A")) @@ -477,7 +477,7 @@ def testReadAllFlowObjectsWithMinCreateTime(self): def testReadAllFlowObjectsWithMaxCreateTime(self): client_id_1 = "C.1111111111111111" - self.db.WriteClientMetadata(client_id_1, fleetspeak_enabled=False) + self.db.WriteClientMetadata(client_id_1) self.db.WriteFlowObject( rdf_flow_objects.Flow(client_id=client_id_1, flow_id="0000001A")) @@ -493,8 +493,8 @@ def testReadAllFlowObjectsWithMaxCreateTime(self): def testReadAllFlowObjectsWithClientID(self): client_id_1 = "C.1111111111111111" client_id_2 = "C.2222222222222222" - self.db.WriteClientMetadata(client_id_1, fleetspeak_enabled=False) - self.db.WriteClientMetadata(client_id_2, fleetspeak_enabled=False) + self.db.WriteClientMetadata(client_id_1) + self.db.WriteClientMetadata(client_id_2) self.db.WriteFlowObject( rdf_flow_objects.Flow(client_id=client_id_1, flow_id="0000001A")) @@ -556,7 +556,7 @@ def testReadAllFlowObjectsWithParentFlowIDWithoutChildren(self): def testReadAllFlowObjectsWithoutChildren(self): client_id_1 = "C.1111111111111111" - self.db.WriteClientMetadata(client_id_1, fleetspeak_enabled=False) + self.db.WriteClientMetadata(client_id_1) self.db.WriteFlowObject( rdf_flow_objects.Flow(client_id=client_id_1, flow_id="0000001A")) @@ -571,7 +571,7 @@ def testReadAllFlowObjectsWithoutChildren(self): def testReadAllFlowObjectsWithNotCreatedBy(self): client_id_1 = "C.1111111111111111" - self.db.WriteClientMetadata(client_id_1, fleetspeak_enabled=False) + self.db.WriteClientMetadata(client_id_1) self.db.WriteFlowObject( rdf_flow_objects.Flow( @@ -589,8 +589,8 @@ def testReadAllFlowObjectsWithNotCreatedBy(self): def testReadAllFlowObjectsWithAllConditions(self): client_id_1 = "C.1111111111111111" client_id_2 = "C.2222222222222222" - self.db.WriteClientMetadata(client_id_1, fleetspeak_enabled=False) - self.db.WriteClientMetadata(client_id_2, fleetspeak_enabled=False) + self.db.WriteClientMetadata(client_id_1) + self.db.WriteClientMetadata(client_id_2) min_timestamp = self.db.Now() @@ -737,7 +737,7 @@ def testRequestWriting(self): client_id=client_id_1, flow_id=flow_id_1) ]) for client_id in [client_id_1, client_id_2]: - self.db.WriteClientMetadata(client_id, fleetspeak_enabled=False) + self.db.WriteClientMetadata(client_id) requests = [] for flow_id in [flow_id_1, flow_id_2]: @@ -1644,6 +1644,36 @@ def testReleaseProcessedFlow(self): self.assertFalse(self.db.ReleaseProcessedFlow(processed_flow)) + def testReleaseProcessedFlowWithRequestScheduledInFuture(self): + client_id = db_test_utils.InitializeClient(self.db) + flow_id = db_test_utils.InitializeFlow( + self.db, client_id, next_request_to_process=2 + ) + + processing_time = rdfvalue.Duration.From(60, rdfvalue.SECONDS) + + processed_flow = self.db.LeaseFlowForProcessing( + client_id, flow_id, processing_time + ) + + # Let's say we processed one request on this flow. + processed_flow.next_request_to_process = 2 + + # There is a request ready for processing but only in the future. + # It shouldn't be returned as ready for processing. + self.db.WriteFlowRequests([ + rdf_flow_objects.FlowRequest( + client_id=client_id, + flow_id=flow_id, + request_id=2, + start_time=rdfvalue.RDFDatetime.Now() + + rdfvalue.Duration.From(60, rdfvalue.SECONDS), + needs_processing=True, + ), + ]) + + self.assertTrue(self.db.ReleaseProcessedFlow(processed_flow)) + def testReleaseProcessedFlowWithProcessedFlowRequest(self): client_id = db_test_utils.InitializeClient(self.db) flow_id = db_test_utils.InitializeFlow(self.db, client_id) @@ -1663,7 +1693,7 @@ def testReleaseProcessedFlowWithProcessedFlowRequest(self): def testReadChildFlows(self): client_id = u"C.1234567890123456" - self.db.WriteClientMetadata(client_id, fleetspeak_enabled=False) + self.db.WriteClientMetadata(client_id) self.db.WriteFlowObject( rdf_flow_objects.Flow(flow_id=u"00000001", client_id=client_id)) @@ -1684,7 +1714,7 @@ def testReadChildFlows(self): parent_flow_id=u"00000001")) # This one is completely unrelated (different client id). - self.db.WriteClientMetadata(u"C.1234567890123457", fleetspeak_enabled=False) + self.db.WriteClientMetadata("C.1234567890123457") self.db.WriteFlowObject( rdf_flow_objects.Flow( flow_id=u"00000001", client_id=u"C.1234567890123457")) @@ -1737,8 +1767,8 @@ def testDeleteAllFlowRequestsAndResponses(self): flow_id1 = u"1234ABCD" flow_id2 = u"1234ABCE" - self.db.WriteClientMetadata(client_id1, fleetspeak_enabled=True) - self.db.WriteClientMetadata(client_id2, fleetspeak_enabled=True) + self.db.WriteClientMetadata(client_id1) + self.db.WriteClientMetadata(client_id2) self._WriteRequestAndResponses(client_id1, flow_id1) self._WriteRequestAndResponses(client_id1, flow_id2) @@ -1763,7 +1793,7 @@ def testDeleteAllFlowRequestsAndResponsesWithClientRequests(self): client_id = u"C.1234567890123456" flow_id = u"1234ABCD" - self.db.WriteClientMetadata(client_id, fleetspeak_enabled=True) + self.db.WriteClientMetadata(client_id) self._WriteRequestAndResponses(client_id, flow_id) @@ -1998,6 +2028,108 @@ def Callback(request): leftover = self.db.ReadFlowProcessingRequests() self.assertEqual(leftover, []) + def testFlowRequestsStartTimeIsRespectedWhenResponsesAreWritten(self): + client_id = db_test_utils.InitializeClient(self.db) + flow_id = db_test_utils.InitializeFlow(self.db, client_id) + + request_queue = queue.Queue() + + def Callback(request): + self.db.AckFlowProcessingRequests([request]) + request_queue.put(request) + + self.db.RegisterFlowProcessingHandler(Callback) + self.addCleanup(self.db.UnregisterFlowProcessingHandler) + + now = rdfvalue.RDFDatetime.Now() + delivery_time = rdfvalue.RDFDatetime.FromSecondsSinceEpoch( + now.AsSecondsSinceEpoch() + 5 + ) + request = rdf_flow_objects.FlowRequest( + client_id=client_id, + flow_id=flow_id, + request_id=1, + start_time=delivery_time, + nr_responses_expected=1, + next_state="Foo", + needs_processing=True, + ) + self.db.WriteFlowRequests([request]) + + response = rdf_flow_objects.FlowResponse( + client_id=client_id, + flow_id=flow_id, + request_id=request.request_id, + response_id=0, + # For the purpose of the test, the payload can be arbitrary, + # using rdf_flows.ClientActionRequest as a sample struct. + payload=rdf_flows.ClientActionRequest(), + ) + self.db.WriteFlowResponses([response]) + + try: + l = request_queue.get(True, timeout=3) + self.fail("Expected to get no messages within 3 seconds, got 1") + except queue.Empty: + pass + + try: + l = request_queue.get(True, timeout=10) + except queue.Empty: + self.fail("Timed out waiting for messages") + + self.assertGreater(rdfvalue.RDFDatetime.Now(), l.delivery_time) + + def testFlowProcessingRequestIsAlwaysWrittenIfStartTimeIsSpecified(self): + client_id = db_test_utils.InitializeClient(self.db) + flow_id = db_test_utils.InitializeFlow(self.db, client_id) + + fprs = self.db.ReadFlowProcessingRequests() + self.assertEmpty(fprs) + + now = rdfvalue.RDFDatetime.Now() + delivery_time = now + rdfvalue.Duration("5s") + request = rdf_flow_objects.FlowRequest( + client_id=client_id, + flow_id=flow_id, + # Note that the request_id is different from the next_request_id + # recorded in the flow object (with a freshly initialized flow that + # will be 1). For flow requests with start_time set, we have to make + # sure that a FlowProcessingRequest is written to the queue so that + # the flow would "wake up" when needed - by that time the + # next_request_id might be equivalent to the FlowRequest's request_id + # and the state will get processed. + request_id=42, + start_time=delivery_time, + nr_responses_expected=1, + next_state="Foo", + needs_processing=True, + ) + self.db.WriteFlowRequests([request]) + + fprs = self.db.ReadFlowProcessingRequests() + self.assertLen(fprs, 1) + + def testFPRNotWrittenIfStartTimeNotSpecifiedAndIdDoesNotMatch(self): + client_id = db_test_utils.InitializeClient(self.db) + flow_id = db_test_utils.InitializeFlow(self.db, client_id) + + fprs = self.db.ReadFlowProcessingRequests() + self.assertEmpty(fprs) + + request = rdf_flow_objects.FlowRequest( + client_id=client_id, + flow_id=flow_id, + request_id=42, + nr_responses_expected=1, + next_state="Foo", + needs_processing=True, + ) + self.db.WriteFlowRequests([request]) + + fprs = self.db.ReadFlowProcessingRequests() + self.assertEmpty(fprs) + def testIncrementalFlowProcessingRequests(self): pass @@ -3114,19 +3246,16 @@ def testCountFlowOutputPluginLogEntriesRespectsWithTypeFilter(self): def _SetupScheduledFlow(self, **kwargs): merged_kwargs = { - "scheduled_flow_id": - flow.RandomFlowId(), - "flow_name": - file.CollectSingleFile.__name__, - "flow_args": - rdf_file_finder.CollectSingleFileArgs( - max_size_bytes=random.randint(0, 10)), - "runner_args": - rdf_flow_runner.FlowRunnerArgs( - network_bytes_limit=random.randint(0, 10)), - "create_time": - rdfvalue.RDFDatetime.Now(), - **kwargs + "scheduled_flow_id": flow.RandomFlowId(), + "flow_name": file.CollectFilesByKnownPath.__name__, + "flow_args": rdf_file_finder.CollectFilesByKnownPathArgs( + collection_level=random.randint(0, 3) + ), + "runner_args": rdf_flow_runner.FlowRunnerArgs( + network_bytes_limit=random.randint(0, 10) + ), + "create_time": rdfvalue.RDFDatetime.Now(), + **kwargs, } sf = rdf_flow_objects.ScheduledFlow(**merged_kwargs) @@ -3146,11 +3275,12 @@ def testWriteScheduledFlowPersistsAllFields(self): client_id=client_id, creator=username, scheduled_flow_id="1234123421342134", - flow_name=file.CollectSingleFile.__name__, - flow_args=rdf_file_finder.CollectSingleFileArgs(path="/baz"), + flow_name=file.CollectFilesByKnownPath.__name__, + flow_args=rdf_file_finder.CollectFilesByKnownPathArgs(paths=["/baz"]), runner_args=rdf_flow_runner.FlowRunnerArgs(network_bytes_limit=1024), create_time=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(42), - error="foobazzle disintegrated") + error="foobazzle disintegrated", + ) results = self.db.ListScheduledFlows(client_id, username) self.assertEqual([sf], results) @@ -3159,8 +3289,8 @@ def testWriteScheduledFlowPersistsAllFields(self): self.assertEqual(result.client_id, client_id) self.assertEqual(result.creator, username) self.assertEqual(result.scheduled_flow_id, "1234123421342134") - self.assertEqual(result.flow_name, file.CollectSingleFile.__name__) - self.assertEqual(result.flow_args.path, "/baz") + self.assertEqual(result.flow_name, file.CollectFilesByKnownPath.__name__) + self.assertEqual(result.flow_args.paths, ["/baz"]) self.assertEqual(result.runner_args.network_bytes_limit, 1024) self.assertEqual(result.create_time, rdfvalue.RDFDatetime.FromSecondsSinceEpoch(42)) diff --git a/grr/server/grr_response_server/databases/db_hunts_test.py b/grr/server/grr_response_server/databases/db_hunts_test.py index e0f5cbdba0..39cf8d6f4f 100644 --- a/grr/server/grr_response_server/databases/db_hunts_test.py +++ b/grr/server/grr_response_server/databases/db_hunts_test.py @@ -34,7 +34,7 @@ def _SetupHuntClientAndFlow(self, client_id = db_test_utils.InitializeClient(self.db, client_id=client_id) # Top-level hunt-induced flows should have hunt's id. flow_id = flow_id or hunt_id - self.db.WriteClientMetadata(client_id, fleetspeak_enabled=False) + self.db.WriteClientMetadata(client_id) rdf_flow = rdf_flow_objects.Flow( client_id=client_id, @@ -179,6 +179,28 @@ def testDeletingHuntObjectWorks(self): with self.assertRaises(db.UnknownHuntError): self.db.ReadHuntObject(hunt_obj.hunt_id) + def testDeleteHuntObjectWithApprovalRequest(self): + creator = db_test_utils.InitializeUser(self.db) + approver = db_test_utils.InitializeUser(self.db) + hunt_id = db_test_utils.InitializeHunt(self.db, creator=creator) + + approval = rdf_objects.ApprovalRequest() + approval.approval_type = ( + rdf_objects.ApprovalRequest.ApprovalType.APPROVAL_TYPE_HUNT + ) + approval.requestor_username = creator + approval.notified_users = [approver] + approval.subject_id = hunt_id + approval.expiration_time = ( + rdfvalue.RDFDatetime.Now() + rdfvalue.Duration.From(1, rdfvalue.DAYS) + ) + approval_id = self.db.WriteApprovalRequest(approval) + + self.db.DeleteHuntObject(hunt_id=hunt_id) + + with self.assertRaises(db.UnknownApprovalRequestError): + self.db.ReadApprovalRequest(creator, approval_id) + def testReadHuntObjectsReturnsEmptyListWhenNoHunts(self): self.assertEqual(self.db.ReadHuntObjects(offset=0, count=db.MAX_COUNT), []) diff --git a/grr/server/grr_response_server/databases/db_test_utils.py b/grr/server/grr_response_server/databases/db_test_utils.py index c2f84a78a6..98cafe513b 100644 --- a/grr/server/grr_response_server/databases/db_test_utils.py +++ b/grr/server/grr_response_server/databases/db_test_utils.py @@ -4,13 +4,13 @@ import itertools import random import string - from typing import Any, Callable, Dict, Iterable, Optional, Text from grr_response_server.databases import db as abstract_db from grr_response_server.rdfvalues import cronjobs as rdf_cronjobs from grr_response_server.rdfvalues import flow_objects as rdf_flow_objects from grr_response_server.rdfvalues import hunt_objects as rdf_hunt_objects +from grr_response_proto.rrg import startup_pb2 as rrg_startup_pb2 class QueryTestHelpersMixin(object): @@ -178,7 +178,32 @@ def InitializeClient( for _ in range(16): client_id += random.choice("0123456789abcdef") - db.WriteClientMetadata(client_id, fleetspeak_enabled=False) + db.WriteClientMetadata(client_id) + return client_id + + +def InitializeRRGClient( + db: abstract_db.Database, + client_id: Optional[str] = None, +) -> str: + """Initialize a test client that supports RRG. + + Args: + db: A database object. + client_id: A specific client id to use for initialized client. If none is + provided a randomly generated one is used. + + Returns: + A client id for the initialized client. + """ + client_id = InitializeClient(db, client_id) + + startup = rrg_startup_pb2.Startup() + startup.metadata.version.major = 1 + startup.metadata.version.minor = 2 + startup.metadata.version.patch = 3 + db.WriteClientRRGStartup(client_id, startup) + return client_id diff --git a/grr/server/grr_response_server/databases/db_test_utils_test.py b/grr/server/grr_response_server/databases/db_test_utils_test.py index 81c6600b96..719a0d7576 100644 --- a/grr/server/grr_response_server/databases/db_test_utils_test.py +++ b/grr/server/grr_response_server/databases/db_test_utils_test.py @@ -180,6 +180,16 @@ def testSupplied(self): self.assertIsNotNone(db.ReadClientMetadata(client_id)) +class InitializeRRGClientTest(absltest.TestCase): + + def testRandom(self): + db = mem_db.InMemoryDB() + + client_id = db_test_utils.InitializeRRGClient(db) + self.assertIsNotNone(db.ReadClientMetadata(client_id)) + self.assertIsNotNone(db.ReadClientRRGStartup(client_id)) + + class InitializeUserTest(absltest.TestCase): def testRandom(self): diff --git a/grr/server/grr_response_server/databases/mem_clients.py b/grr/server/grr_response_server/databases/mem_clients.py index a2dec5d9a9..8b6e138a97 100644 --- a/grr/server/grr_response_server/databases/mem_clients.py +++ b/grr/server/grr_response_server/databases/mem_clients.py @@ -26,23 +26,18 @@ def WriteClientMetadata( self, client_id: str, certificate: Optional[rdf_crypto.RDFX509Cert] = None, - fleetspeak_enabled: Optional[bool] = None, first_seen: Optional[rdfvalue.RDFDatetime] = None, last_ping: Optional[rdfvalue.RDFDatetime] = None, last_clock: Optional[rdfvalue.RDFDatetime] = None, last_ip: Optional[rdf_client_network.NetworkAddress] = None, last_foreman: Optional[rdfvalue.RDFDatetime] = None, fleetspeak_validation_info: Optional[Mapping[str, str]] = None, - rrg_support: Optional[bool] = None, ) -> None: """Write metadata about the client.""" md = {} if certificate is not None: md["certificate"] = certificate - if fleetspeak_enabled is not None: - md["fleetspeak_enabled"] = fleetspeak_enabled - if first_seen is not None: md["first_seen"] = first_seen @@ -66,9 +61,6 @@ def WriteClientMetadata( # Write null for empty or non-existent validation info. md["last_fleetspeak_validation_info"] = None - if rrg_support is not None: - md["rrg_support"] = rrg_support - self.metadatas.setdefault(client_id, {}).update(md) @utils.Synchronized @@ -82,7 +74,6 @@ def MultiReadClientMetadata(self, client_ids): metadata = rdf_objects.ClientMetadata( certificate=md.get("certificate"), - fleetspeak_enabled=md.get("fleetspeak_enabled"), first_seen=md.get("first_seen"), ping=md.get("ping"), clock=md.get("clock"), @@ -90,7 +81,6 @@ def MultiReadClientMetadata(self, client_ids): last_foreman_time=md.get("last_foreman_time"), last_crash_timestamp=md.get("last_crash_timestamp"), startup_info_timestamp=md.get("startup_info_timestamp"), - rrg_support=bool(md.get("rrg_support")), ) fsvi = md.get("last_fleetspeak_validation_info") @@ -179,20 +169,15 @@ def MultiReadClientFullInfo(self, client_ids, min_last_ping=None): def ReadClientLastPings(self, min_last_ping=None, max_last_ping=None, - fleetspeak_enabled=None, batch_size=db.CLIENT_IDS_BATCH_SIZE): """Yields dicts of last-ping timestamps for clients in the DB.""" last_pings = {} for client_id, metadata in self.metadatas.items(): last_ping = metadata.get("ping", rdfvalue.RDFDatetime(0)) - is_fleetspeak_client = metadata.get("fleetspeak_enabled", False) if min_last_ping is not None and last_ping < min_last_ping: continue elif max_last_ping is not None and last_ping > max_last_ping: continue - elif (fleetspeak_enabled is not None and - is_fleetspeak_client != fleetspeak_enabled): - continue else: last_pings[client_id] = metadata.get("ping", None) @@ -350,6 +335,20 @@ def WriteClientRRGStartup( self.rrg_startups[client_id].append(startup_copy) + @utils.Synchronized + def ReadClientRRGStartup( + self, + client_id: str, + ) -> Optional[rrg_startup_pb2.Startup]: + """Reads the latest RRG startup entry for the given client.""" + if client_id not in self.metadatas: + raise db.UnknownClientError(client_id) + + try: + return self.rrg_startups[client_id][-1] + except IndexError: + return None + @utils.Synchronized def ReadClientStartupInfo(self, client_id: str) -> Optional[rdf_client.StartupInfo]: diff --git a/grr/server/grr_response_server/databases/mem_cronjobs.py b/grr/server/grr_response_server/databases/mem_cronjobs.py index e3f1b4fb48..62940d0b63 100644 --- a/grr/server/grr_response_server/databases/mem_cronjobs.py +++ b/grr/server/grr_response_server/databases/mem_cronjobs.py @@ -5,6 +5,7 @@ from grr_response_core.lib import rdfvalue from grr_response_core.lib import utils from grr_response_server.databases import db +from grr_response_server.rdfvalues import objects as rdf_objects class InMemoryDBCronJobMixin(object): @@ -89,6 +90,19 @@ def DeleteCronJob(self, cronjob_id): for job_run in self.ReadCronJobRuns(cronjob_id): del self.cronjob_runs[(cronjob_id, job_run.run_id)] + for approvals in self.approvals_by_username.values(): + # We use `list` around dictionary items iterator to avoid errors about + # dictionary modification during iteration. + for approval_id, approval in list(approvals.items()): + if ( + approval.approval_type + != rdf_objects.ApprovalRequest.ApprovalType.APPROVAL_TYPE_CRON_JOB + ): + continue + if approval.subject_id != cronjob_id: + continue + del approvals[approval_id] + @utils.Synchronized def LeaseCronJobs(self, cronjob_ids=None, lease_time=None): """Leases all available cron jobs.""" diff --git a/grr/server/grr_response_server/databases/mem_flows.py b/grr/server/grr_response_server/databases/mem_flows.py index b281a1880f..ef8051deb8 100644 --- a/grr/server/grr_response_server/databases/mem_flows.py +++ b/grr/server/grr_response_server/databases/mem_flows.py @@ -359,7 +359,10 @@ def WriteFlowRequests(self, requests): if request.needs_processing: flow = self.flows[(request.client_id, request.flow_id)] - if flow.next_request_to_process == request.request_id: + if ( + flow.next_request_to_process == request.request_id + or request.start_time is not None + ): flow_processing_requests.append( rdf_flows.FlowProcessingRequest( client_id=request.client_id, @@ -593,9 +596,13 @@ def ReleaseProcessedFlow(self, flow_obj): key = (flow_obj.client_id, flow_obj.flow_id) next_id_to_process = flow_obj.next_request_to_process request_dict = self.flow_requests.get(key, {}) - if (next_id_to_process in request_dict and - request_dict[next_id_to_process].needs_processing): - return False + if ( + next_id_to_process in request_dict + and request_dict[next_id_to_process].needs_processing + ): + start_time = request_dict[next_id_to_process].start_time + if start_time is None or start_time < rdfvalue.RDFDatetime.Now(): + return False self.UpdateFlow( flow_obj.client_id, diff --git a/grr/server/grr_response_server/databases/mem_hunts.py b/grr/server/grr_response_server/databases/mem_hunts.py index b3cfc31bdf..ef6f82cc12 100644 --- a/grr/server/grr_response_server/databases/mem_hunts.py +++ b/grr/server/grr_response_server/databases/mem_hunts.py @@ -11,6 +11,7 @@ from grr_response_server.rdfvalues import flow_objects as rdf_flow_objects from grr_response_server.rdfvalues import flow_runner as rdf_flow_runner from grr_response_server.rdfvalues import hunt_objects as rdf_hunt_objects +from grr_response_server.rdfvalues import objects as rdf_objects class InMemoryDBHuntMixin(object): @@ -102,11 +103,25 @@ def UpdateHuntOutputPluginState(self, hunt_id, state_index, update_fn): @utils.Synchronized def DeleteHuntObject(self, hunt_id): + """Deletes a hunt object with a given id.""" try: del self.hunts[hunt_id] except KeyError: raise db.UnknownHuntError(hunt_id) + for approvals in self.approvals_by_username.values(): + # We use `list` around dictionary items iterator to avoid errors about + # dictionary modification during iteration. + for approval_id, approval in list(approvals.items()): + if ( + approval.approval_type + != rdf_objects.ApprovalRequest.ApprovalType.APPROVAL_TYPE_HUNT + ): + continue + if approval.subject_id != hunt_id: + continue + del approvals[approval_id] + @utils.Synchronized def ReadHuntObject(self, hunt_id): """Reads a hunt object from the database.""" diff --git a/grr/server/grr_response_server/databases/mysql_clients.py b/grr/server/grr_response_server/databases/mysql_clients.py index 10fc5f0e8e..a84f865669 100644 --- a/grr/server/grr_response_server/databases/mysql_clients.py +++ b/grr/server/grr_response_server/databases/mysql_clients.py @@ -30,14 +30,12 @@ def WriteClientMetadata( self, client_id: str, certificate: Optional[rdf_crypto.RDFX509Cert] = None, - fleetspeak_enabled: Optional[bool] = None, first_seen: Optional[rdfvalue.RDFDatetime] = None, last_ping: Optional[rdfvalue.RDFDatetime] = None, last_clock: Optional[rdfvalue.RDFDatetime] = None, last_ip: Optional[rdf_client_network.NetworkAddress] = None, last_foreman: Optional[rdfvalue.RDFDatetime] = None, fleetspeak_validation_info: Optional[Mapping[str, str]] = None, - rrg_support: Optional[bool] = None, cursor: Optional[MySQLdb.cursors.Cursor] = None, ) -> None: """Write metadata about the client.""" @@ -50,9 +48,6 @@ def WriteClientMetadata( if certificate: placeholders.append("%(certificate)s") values["certificate"] = certificate.SerializeToBytes() - if fleetspeak_enabled is not None: - placeholders.append("%(fleetspeak_enabled)s") - values["fleetspeak_enabled"] = fleetspeak_enabled if first_seen is not None: placeholders.append("FROM_UNIXTIME(%(first_seen)s)") values["first_seen"] = mysql_utils.RDFDatetimeToTimestamp(first_seen) @@ -68,9 +63,6 @@ def WriteClientMetadata( if last_foreman: placeholders.append("FROM_UNIXTIME(%(last_foreman)s)") values["last_foreman"] = mysql_utils.RDFDatetimeToTimestamp(last_foreman) - if rrg_support is not None: - placeholders.append("%(rrg_support)s") - values["rrg_support"] = rrg_support placeholders.append("%(last_fleetspeak_validation_info)s") if fleetspeak_validation_info: @@ -103,7 +95,6 @@ def MultiReadClientMetadata(self, client_ids, cursor=None): query = """ SELECT client_id, - fleetspeak_enabled, certificate, UNIX_TIMESTAMP(last_ping), UNIX_TIMESTAMP(last_clock), @@ -112,8 +103,7 @@ def MultiReadClientMetadata(self, client_ids, cursor=None): UNIX_TIMESTAMP(first_seen), UNIX_TIMESTAMP(last_crash_timestamp), UNIX_TIMESTAMP(last_startup_timestamp), - last_fleetspeak_validation_info, - rrg_support + last_fleetspeak_validation_info FROM clients WHERE @@ -124,10 +114,9 @@ def MultiReadClientMetadata(self, client_ids, cursor=None): row = cursor.fetchone() if not row: break - cid, fs, crt, ping, clk, ip, foreman, first, lct, lst, fsvi, rrg = row + cid, crt, ping, clk, ip, foreman, first, lct, lst, fsvi = row metadata = rdf_objects.ClientMetadata( certificate=crt, - fleetspeak_enabled=fs, first_seen=mysql_utils.TimestampToRDFDatetime(first), ping=mysql_utils.TimestampToRDFDatetime(ping), clock=mysql_utils.TimestampToRDFDatetime(clk), @@ -137,7 +126,6 @@ def MultiReadClientMetadata(self, client_ids, cursor=None): last_foreman_time=mysql_utils.TimestampToRDFDatetime(foreman), startup_info_timestamp=mysql_utils.TimestampToRDFDatetime(lst), last_crash_timestamp=mysql_utils.TimestampToRDFDatetime(lct), - rrg_support=rrg, ) if fsvi: @@ -381,6 +369,40 @@ def WriteClientRRGStartup( except MySQLdb.IntegrityError as error: raise db.UnknownClientError(client_id) from error + @mysql_utils.WithTransaction() + def ReadClientRRGStartup( + self, + client_id: str, + cursor: Optional[MySQLdb.cursors.Cursor] = None, + ) -> Optional[rrg_startup_pb2.Startup]: + """Reads the latest RRG startup entry for the given client.""" + query = """ + SELECT su.startup + FROM clients + LEFT JOIN (SELECT startup + FROM client_rrg_startup_history + WHERE client_id = %(client_id)s + ORDER BY timestamp DESC + LIMIT 1) AS su + ON TRUE + WHERE client_id = %(client_id)s + """ + params = { + "client_id": db_utils.ClientIDToInt(client_id), + } + + cursor.execute(query, params) + + row = cursor.fetchone() + if row is None: + raise db.UnknownClientError(client_id) + + (startup_bytes,) = row + if startup_bytes is None: + return None + + return rrg_startup_pb2.Startup.FromString(startup_bytes) + @mysql_utils.WithTransaction(readonly=True) def ReadClientStartupInfo( self, @@ -452,7 +474,6 @@ def _ResponseToClientsFullInfo(self, response): for row in response: ( cid, - fs, crt, ip, ping, @@ -476,7 +497,6 @@ def _ResponseToClientsFullInfo(self, response): metadata = rdf_objects.ClientMetadata( certificate=crt, - fleetspeak_enabled=fs, first_seen=mysql_utils.TimestampToRDFDatetime(first), ping=mysql_utils.TimestampToRDFDatetime(ping), clock=mysql_utils.TimestampToRDFDatetime(clk), @@ -541,7 +561,7 @@ def MultiReadClientFullInfo(self, return {} query = """ - SELECT c.client_id, c.fleetspeak_enabled, c.certificate, c.last_ip, + SELECT c.client_id, c.certificate, c.last_ip, UNIX_TIMESTAMP(c.last_ping), UNIX_TIMESTAMP(c.last_clock), UNIX_TIMESTAMP(c.last_foreman), @@ -585,7 +605,6 @@ def MultiReadClientFullInfo(self, def ReadClientLastPings(self, min_last_ping=None, max_last_ping=None, - fleetspeak_enabled=None, batch_size=db.CLIENT_IDS_BATCH_SIZE): """Yields dicts of last-ping timestamps for clients in the DB.""" last_client_id = db_utils.IntToClientID(0) @@ -596,7 +615,7 @@ def ReadClientLastPings(self, batch_size, min_last_ping=min_last_ping, max_last_ping=max_last_ping, - fleetspeak_enabled=fleetspeak_enabled) + ) if last_pings: yield last_pings if len(last_pings) < batch_size: @@ -608,7 +627,6 @@ def _ReadClientLastPings(self, count, min_last_ping=None, max_last_ping=None, - fleetspeak_enabled=None, cursor=None): """Yields dicts of last-ping timestamps for clients in the DB.""" where_filters = ["client_id > %s"] @@ -620,12 +638,6 @@ def _ReadClientLastPings(self, where_filters.append( "(last_ping IS NULL OR last_ping <= FROM_UNIXTIME(%s))") query_values.append(mysql_utils.RDFDatetimeToTimestamp(max_last_ping)) - if fleetspeak_enabled is not None: - if fleetspeak_enabled: - where_filters.append("fleetspeak_enabled IS TRUE") - else: - where_filters.append( - "(fleetspeak_enabled IS NULL OR fleetspeak_enabled IS FALSE)") query = """ SELECT client_id, UNIX_TIMESTAMP(last_ping) diff --git a/grr/server/grr_response_server/databases/mysql_cronjobs.py b/grr/server/grr_response_server/databases/mysql_cronjobs.py index 3c525d0ff2..17b56b0412 100644 --- a/grr/server/grr_response_server/databases/mysql_cronjobs.py +++ b/grr/server/grr_response_server/databases/mysql_cronjobs.py @@ -10,6 +10,7 @@ from grr_response_server.databases import db_utils from grr_response_server.databases import mysql_utils from grr_response_server.rdfvalues import cronjobs as rdf_cronjobs +from grr_response_server.rdfvalues import objects as rdf_objects class MySQLDBCronJobMixin(object): @@ -90,10 +91,25 @@ def DisableCronJob(self, cronjob_id, cursor=None): @mysql_utils.WithTransaction() def DeleteCronJob(self, cronjob_id, cursor=None): + """Deletes a cronjob along with all its runs.""" res = cursor.execute("DELETE FROM cron_jobs WHERE job_id=%s", [cronjob_id]) if res != 1: raise db.UnknownCronJobError("CronJob with id %s not found." % cronjob_id) + query = """ + DELETE + FROM approval_request + WHERE approval_type = %(approval_type)s + AND subject_id = %(cron_job_id)s + """ + args = { + "approval_type": int( + rdf_objects.ApprovalRequest.ApprovalType.APPROVAL_TYPE_CRON_JOB + ), + "cron_job_id": cronjob_id, + } + cursor.execute(query, args) + @mysql_utils.WithTransaction() def UpdateCronJob(self, cronjob_id, diff --git a/grr/server/grr_response_server/databases/mysql_flows.py b/grr/server/grr_response_server/databases/mysql_flows.py index 471d912597..e42c163213 100644 --- a/grr/server/grr_response_server/databases/mysql_flows.py +++ b/grr/server/grr_response_server/databases/mysql_flows.py @@ -626,13 +626,21 @@ def WriteFlowRequests(self, requests, cursor=None): if r.needs_processing: needs_processing.setdefault((r.client_id, r.flow_id), []).append(r) + start_time = None + if r.start_time is not None: + start_time = r.start_time.AsDatetime() + flow_keys.append((r.client_id, r.flow_id)) - templates.append("(%s, %s, %s, %s, %s, %s, %s)") + templates.append("(%s, %s, %s, %s, %s, %s, %s, %s)") args.extend([ db_utils.ClientIDToInt(r.client_id), - db_utils.FlowIDToInt(r.flow_id), r.request_id, r.needs_processing, - r.callback_state, r.next_response_id, - r.SerializeToBytes() + db_utils.FlowIDToInt(r.flow_id), + r.request_id, + r.needs_processing, + r.callback_state, + r.next_response_id, + start_time, + r.SerializeToBytes(), ]) if needs_processing: @@ -656,7 +664,10 @@ def WriteFlowRequests(self, requests, cursor=None): flow_id = db_utils.IntToFlowID(flow_id_int) candidate_requests = needs_processing.get((client_id, flow_id), []) for r in candidate_requests: - if next_request_to_process == r.request_id: + if ( + next_request_to_process == r.request_id + or r.start_time is not None + ): flow_processing_requests.append( rdf_flows.FlowProcessingRequest( client_id=client_id, @@ -666,10 +677,12 @@ def WriteFlowRequests(self, requests, cursor=None): if flow_processing_requests: self._WriteFlowProcessingRequests(flow_processing_requests, cursor) - query = ("INSERT INTO flow_requests " - "(client_id, flow_id, request_id, needs_processing, " - "callback_state, next_response_id, request) " - "VALUES ") + query = ( + "INSERT INTO flow_requests " + "(client_id, flow_id, request_id, needs_processing, " + "callback_state, next_response_id, start_time, request) " + "VALUES " + ) query += ", ".join(templates) try: cursor.execute(query, args) @@ -1128,6 +1141,7 @@ def ReleaseProcessedFlow(self, flow_obj, cursor=None): client_id = %(client_id)s AND flow_id = %(flow_id)s AND request_id = %(next_request_to_process)s AND + (start_time IS NULL OR start_time < NOW(6)) AND needs_processing ) AS needs_processing ON diff --git a/grr/server/grr_response_server/databases/mysql_hunts.py b/grr/server/grr_response_server/databases/mysql_hunts.py index f0d2e8ce2c..ec090ff572 100644 --- a/grr/server/grr_response_server/databases/mysql_hunts.py +++ b/grr/server/grr_response_server/databases/mysql_hunts.py @@ -73,16 +73,19 @@ def WriteHuntObject(self, hunt_obj, cursor=None): raise db.DuplicatedHuntError(hunt_id=hunt_obj.hunt_id, cause=error) @mysql_utils.WithTransaction() - def UpdateHuntObject(self, - hunt_id, - duration=None, - client_rate=None, - client_limit=None, - hunt_state=None, - hunt_state_comment=None, - start_time=None, - num_clients_at_start_time=None, - cursor=None): + def UpdateHuntObject( + self, + hunt_id, + duration=None, + client_rate=None, + client_limit=None, + hunt_state=None, + hunt_state_reason=None, + hunt_state_comment=None, + start_time=None, + num_clients_at_start_time=None, + cursor=None, + ): """Updates the hunt object by applying the update function.""" vals = [] args = {} @@ -103,6 +106,10 @@ def UpdateHuntObject(self, vals.append("hunt_state = %(hunt_state)s") args["hunt_state"] = int(hunt_state) + if hunt_state_reason is not None: + vals.append("hunt_state_reason = %(hunt_state_reason)s") + args["hunt_state_reason"] = int(hunt_state_reason) + if hunt_state_comment is not None: vals.append("hunt_state_comment = %(hunt_state_comment)s") args["hunt_state_comment"] = hunt_state_comment @@ -146,6 +153,20 @@ def DeleteHuntObject(self, hunt_id, cursor=None): query = "DELETE FROM hunt_output_plugins_states WHERE hunt_id = %s" cursor.execute(query, [hunt_id_int]) + query = """ + DELETE + FROM approval_request + WHERE approval_type = %(approval_type)s + AND subject_id = %(hunt_id)s + """ + args = { + "approval_type": int( + rdf_objects.ApprovalRequest.ApprovalType.APPROVAL_TYPE_HUNT + ), + "hunt_id": hunt_id, + } + cursor.execute(query, args) + def _HuntObjectFromRow(self, row): """Generates a flow object from a database row.""" ( @@ -442,11 +463,8 @@ def WriteHuntOutputPluginsStates(self, hunt_id, states, cursor=None): columns=columns, placeholders=placeholders)) args = [hunt_id_int, index, state.plugin_descriptor.plugin_name] - # TODO: Stop reading `plugin_args` at all (no fallback). if state.plugin_descriptor.HasField("args"): args.append(state.plugin_descriptor.args.SerializeToBytes()) - elif state.plugin_descriptor.HasField("plugin_args"): - args.append(state.plugin_descriptor.plugin_args.SerializeToBytes()) else: args.append(None) diff --git a/grr/server/grr_response_server/databases/mysql_migrations/0018.sql b/grr/server/grr_response_server/databases/mysql_migrations/0018.sql new file mode 100644 index 0000000000..d24d9100bf --- /dev/null +++ b/grr/server/grr_response_server/databases/mysql_migrations/0018.sql @@ -0,0 +1,2 @@ +ALTER TABLE clients +DROP COLUMN rrg_support; diff --git a/grr/server/grr_response_server/databases/mysql_migrations/0019.sql b/grr/server/grr_response_server/databases/mysql_migrations/0019.sql new file mode 100644 index 0000000000..0bc6ab9a03 --- /dev/null +++ b/grr/server/grr_response_server/databases/mysql_migrations/0019.sql @@ -0,0 +1,2 @@ +ALTER TABLE hunts +ADD COLUMN hunt_state_reason INT UNSIGNED DEFAULT NULL; diff --git a/grr/server/grr_response_server/databases/mysql_migrations/0020.sql b/grr/server/grr_response_server/databases/mysql_migrations/0020.sql new file mode 100644 index 0000000000..ffef78fb54 --- /dev/null +++ b/grr/server/grr_response_server/databases/mysql_migrations/0020.sql @@ -0,0 +1,2 @@ +ALTER TABLE flow_requests +ADD COLUMN start_time timestamp(6) NULL DEFAULT NULL; diff --git a/grr/server/grr_response_server/distro_entry.py b/grr/server/grr_response_server/distro_entry.py index 3d1820b754..1641227a17 100644 --- a/grr/server/grr_response_server/distro_entry.py +++ b/grr/server/grr_response_server/distro_entry.py @@ -27,8 +27,9 @@ def GrrServer(): def GrrFrontend(): - from grr_response_server.bin import frontend - app.run(frontend.main) + from grr_response_server.bin import fleetspeak_frontend + + app.run(fleetspeak_frontend.main) def Worker(): diff --git a/grr/server/grr_response_server/file_store.py b/grr/server/grr_response_server/file_store.py index 1e6585c4f5..b88bc5aaad 100644 --- a/grr/server/grr_response_server/file_store.py +++ b/grr/server/grr_response_server/file_store.py @@ -223,7 +223,7 @@ def Path(self): _BLOBS_READ_BATCH_SIZE = 200 -BLOBS_READ_TIMEOUT = rdfvalue.Duration.From(30, rdfvalue.SECONDS) +BLOBS_READ_TIMEOUT = rdfvalue.Duration.From(120, rdfvalue.SECONDS) def AddFilesWithUnknownHashes( diff --git a/grr/server/grr_response_server/fleetspeak_utils.py b/grr/server/grr_response_server/fleetspeak_utils.py index a4d59bb4d0..a5c31ed6f1 100644 --- a/grr/server/grr_response_server/fleetspeak_utils.py +++ b/grr/server/grr_response_server/fleetspeak_utils.py @@ -12,7 +12,6 @@ from grr_response_core.lib.rdfvalues import flows as rdf_flows from grr_response_core.lib.util import text from grr_response_core.stats import metrics -from grr_response_server import data_store from grr_response_server import fleetspeak_connector from fleetspeak.src.common.proto.fleetspeak import common_pb2 as fs_common_pb2 from fleetspeak.src.common.proto.fleetspeak import system_pb2 as fs_system_pb2 @@ -31,17 +30,6 @@ READ_TOTAL_TIMEOUT = datetime.timedelta(seconds=120) -def IsFleetspeakEnabledClient(grr_id): - """Returns whether the provided GRR id is a Fleetspeak client.""" - if grr_id is None: - return False - - md = data_store.REL_DB.ReadClientMetadata(grr_id) - if not md: - return False - return md.fleetspeak_enabled - - @FLEETSPEAK_CALL_LATENCY.Timed(fields=["InsertMessage"]) def SendGrrMessageThroughFleetspeak(grr_id: str, grr_msg: rdf_flows.GrrMessage) -> None: diff --git a/grr/server/grr_response_server/flow.py b/grr/server/grr_response_server/flow.py index c97accd5f3..4eb8349f66 100644 --- a/grr/server/grr_response_server/flow.py +++ b/grr/server/grr_response_server/flow.py @@ -84,13 +84,9 @@ def GetOutputPluginStates(output_plugins, source=None): for plugin_descriptor in output_plugins: plugin_class = plugin_descriptor.GetPluginClass() try: - # TODO: Stop reading `plugin_args` at all (no fallback). - if plugin_descriptor.HasField("args"): - plugin_args = plugin_descriptor.args - else: - plugin_args = plugin_descriptor.plugin_args _, plugin_state = plugin_class.CreatePluginAndDefaultState( - source_urn=source, args=plugin_args) + source_urn=source, args=plugin_descriptor.args + ) except Exception as e: # pylint: disable=broad-except raise ValueError("Plugin %s failed to initialize (%s)" % (plugin_class, e)) diff --git a/grr/server/grr_response_server/flow_base.py b/grr/server/grr_response_server/flow_base.py index 6224fa0465..68aba3d9c9 100644 --- a/grr/server/grr_response_server/flow_base.py +++ b/grr/server/grr_response_server/flow_base.py @@ -156,6 +156,8 @@ def __init__(self, rdf_flow: rdf_flow_objects.Flow): self._client_os = None self._client_knowledge_base = None self._client_info: Optional[rdf_client.ClientInformation] = None + + self._python_agent_support: Optional[bool] = None self._rrg_support: Optional[bool] = None self._num_replies_per_type_tag = collections.Counter() @@ -176,9 +178,12 @@ def HeartBeat(self) -> None: """New-style flows don't need heart-beat, keeping for compatibility.""" pass - def CallState(self, - next_state: str = "", - start_time: Optional[rdfvalue.RDFDatetime] = None) -> None: + def CallState( + self, + next_state: str = "", + start_time: Optional[rdfvalue.RDFDatetime] = None, + responses: Optional[Sequence[rdf_structs.RDFStruct]] = None, + ): """This method is used to schedule a new state on a different worker. This is basically the same as CallFlow() except we are calling @@ -189,6 +194,7 @@ def CallState(self, start_time: Start the flow at this time. This delays notification for flow processing into the future. Note that the flow may still be processed earlier if there are client responses waiting. + responses: If specified, responses to be passed to the next state. Raises: ValueError: The next state specified does not exist. @@ -196,14 +202,31 @@ def CallState(self, if not getattr(self, next_state): raise ValueError("Next state %s is invalid." % next_state) + request_id = self.GetNextOutboundId() + if responses: + for index, r in enumerate(responses): + wrapped_response = rdf_flow_objects.FlowResponse( + client_id=self.rdf_flow.client_id, + flow_id=self.rdf_flow.flow_id, + request_id=request_id, + response_id=index, + payload=r, + ) + self.flow_responses.append(wrapped_response) + + nr_responses_expected = len(responses) + else: + nr_responses_expected = 0 + flow_request = rdf_flow_objects.FlowRequest( client_id=self.rdf_flow.client_id, flow_id=self.rdf_flow.flow_id, - request_id=self.GetNextOutboundId(), + request_id=request_id, next_state=next_state, start_time=start_time, - needs_processing=True) - + nr_responses_expected=nr_responses_expected, + needs_processing=True, + ) self.flow_requests.append(flow_request) def CallStateInline(self, @@ -870,12 +893,9 @@ def FlushQueuedMessages(self) -> None: if self.client_action_requests: client_id = self.rdf_flow.client_id - if fleetspeak_utils.IsFleetspeakEnabledClient(client_id): - for request in self.client_action_requests: - msg = rdf_flow_objects.GRRMessageFromClientActionRequest(request) - fleetspeak_utils.SendGrrMessageThroughFleetspeak(client_id, msg) - else: - data_store.REL_DB.WriteClientActionRequests(self.client_action_requests) + for request in self.client_action_requests: + msg = rdf_flow_objects.GRRMessageFromClientActionRequest(request) + fleetspeak_utils.SendGrrMessageThroughFleetspeak(client_id, msg) self.client_action_requests = [] @@ -943,11 +963,7 @@ def _ProcessRepliesWithFlowOutputPlugins( self.rdf_flow.output_plugins_states): plugin_descriptor = output_plugin_state.plugin_descriptor output_plugin_cls = plugin_descriptor.GetPluginClass() - # TODO: Stop reading `plugin_args` at all (no fallback). - if plugin_descriptor.HasField("args"): - args = plugin_descriptor.args - else: - args = plugin_descriptor.plugin_args + args = plugin_descriptor.args output_plugin = output_plugin_cls( source_urn=self.rdf_flow.long_flow_id, args=args) @@ -1122,11 +1138,19 @@ def client_info(self) -> rdf_client.ClientInformation: return client_info + @property + def python_agent_support(self) -> bool: + if self._python_agent_support is None: + startup = data_store.REL_DB.ReadClientStartupInfo(self.client_id) + self._python_agent_support = startup is not None + + return self._python_agent_support + @property def rrg_support(self) -> bool: if self._rrg_support is None: - metadata = data_store.REL_DB.ReadClientMetadata(self.client_id) - self._rrg_support = metadata.rrg_support + rrg_startup = data_store.REL_DB.ReadClientRRGStartup(self.client_id) + self._rrg_support = rrg_startup is not None return self._rrg_support diff --git a/grr/server/grr_response_server/flow_base_test.py b/grr/server/grr_response_server/flow_base_test.py index 783ab96b08..4e9c71fddd 100644 --- a/grr/server/grr_response_server/flow_base_test.py +++ b/grr/server/grr_response_server/flow_base_test.py @@ -87,9 +87,34 @@ def testClientInfoDefault(self, db: abstract_db.Database): self.assertEmpty(flow.client_info.client_name) @db_test_lib.WithDatabase - def testRrgSupport(self, db: abstract_db.Database): + def testPythonAgentSupportFalse(self, db: abstract_db.Database): + client_id = db_test_utils.InitializeRRGClient(db) + + flow = rdf_flow_objects.Flow() + flow.client_id = client_id + flow.flow_id = self._FLOW_ID + + flow = FlowBaseTest.Flow(flow) + self.assertFalse(flow.python_agent_support) + + @db_test_lib.WithDatabase + def testPythoAgentSupportTrue(self, db: abstract_db.Database): client_id = db_test_utils.InitializeClient(db) - db.WriteClientMetadata(client_id, rrg_support=True) + + startup = rdf_client.StartupInfo() + startup.client_info.client_version = 4321 + db.WriteClientStartupInfo(client_id, startup) + + flow = rdf_flow_objects.Flow() + flow.client_id = client_id + flow.flow_id = self._FLOW_ID + + flow = FlowBaseTest.Flow(flow) + self.assertTrue(flow.python_agent_support) + + @db_test_lib.WithDatabase + def testRrgSupport(self, db: abstract_db.Database): + client_id = db_test_utils.InitializeRRGClient(db) flow = rdf_flow_objects.Flow() flow.client_id = client_id @@ -273,8 +298,6 @@ def testCallRRGUnsupported(self, db: abstract_db.Database): client_id = db_test_utils.InitializeClient(db) flow_id = db_test_utils.InitializeFlow(db, client_id) - db.WriteClientMetadata(client_id, rrg_support=False) - rdf_flow = rdf_flow_objects.Flow() rdf_flow.client_id = client_id rdf_flow.flow_id = flow_id @@ -286,11 +309,9 @@ def testCallRRGUnsupported(self, db: abstract_db.Database): @db_test_lib.WithDatabase def testCallRRGSupported(self, db: abstract_db.Database): - client_id = db_test_utils.InitializeClient(db) + client_id = db_test_utils.InitializeRRGClient(db) flow_id = db_test_utils.InitializeFlow(db, client_id) - db.WriteClientMetadata(client_id, rrg_support=True) - rdf_flow = rdf_flow_objects.Flow() rdf_flow.client_id = client_id rdf_flow.flow_id = flow_id diff --git a/grr/server/grr_response_server/flow_test.py b/grr/server/grr_response_server/flow_test.py index 8af4181a62..81fc398017 100644 --- a/grr/server/grr_response_server/flow_test.py +++ b/grr/server/grr_response_server/flow_test.py @@ -17,6 +17,7 @@ from grr_response_server import data_store from grr_response_server import flow from grr_response_server import flow_base +from grr_response_server import flow_responses from grr_response_server import server_stubs from grr_response_server import worker_lib from grr_response_server.databases import db @@ -68,6 +69,44 @@ def ReceiveHello(self, responses): CallStateFlow.success = True +class CallStateFlowWithResponses(flow_base.FlowBase): + """Test flow that calls its own state and passes responses.""" + + # This is a global flag which will be set when the flow runs. + success = False + + def Start(self) -> None: + responses = [ + rdf_paths.PathSpec( + path=f"/tmp/{i}.txt", pathtype=rdf_paths.PathSpec.PathType.OS + ) + for i in range(10) + ] + # Calling the state a little in the future to avoid inline processing done + # by the flow test library. Inline processing will break the CallState + # logic: responses are written after requests, but the inline processing + # is triggered already when requests are written. Inline processing + # doesn't happen if flow requests are scheduled in the future. + self.CallState( + next_state=self.ReceiveHello.__name__, + responses=responses, + start_time=rdfvalue.RDFDatetime.Now() + rdfvalue.Duration("1s"), + ) + + def ReceiveHello(self, responses: flow_responses.Responses) -> None: + if len(responses) != 10: + raise RuntimeError(f"Expected 10 responses, got: {len(responses)}") + + for i, r in enumerate(sorted(responses)): + expected = rdf_paths.PathSpec( + path=f"/tmp/{i}.txt", pathtype=rdf_paths.PathSpec.PathType.OS + ) + if r != expected: + raise RuntimeError(f"Unexpected response: {r}, expected: {expected}") + + CallStateFlowWithResponses.success = True + + class BasicFlowTest(flow_test_lib.FlowTestsBaseclass): def setUp(self): @@ -254,6 +293,15 @@ def testCallState(self): self.assertEqual(CallStateFlow.success, True) + def testCallStateWithResponses(self): + """Test the ability to chain flows.""" + CallStateFlowWithResponses.success = False + # Run the flow in the simulated way + flow_test_lib.StartAndRunFlow( + CallStateFlowWithResponses, client_id=self.client_id + ) + self.assertEqual(CallStateFlow.success, True) + def testChainedFlow(self): """Test the ability to chain flows.""" ParentFlow.success = False @@ -557,14 +605,14 @@ def SetupUser(self, username="u0"): def ScheduleFlow(self, **kwargs): merged_kwargs = { - "flow_name": - file.CollectSingleFile.__name__, - "flow_args": - rdf_file_finder.CollectSingleFileArgs( - path="/foo{}".format(random.randint(0, 1000))), - "runner_args": - rdf_flow_runner.FlowRunnerArgs(cpu_limit=random.randint(0, 60)), - **kwargs + "flow_name": file.CollectFilesByKnownPath.__name__, + "flow_args": rdf_file_finder.CollectFilesByKnownPathArgs( + paths=["/foo{}".format(random.randint(0, 1000))] + ), + "runner_args": rdf_flow_runner.FlowRunnerArgs( + cpu_limit=random.randint(0, 60) + ), + **kwargs, } return flow.ScheduleFlow(**merged_kwargs) @@ -591,9 +639,10 @@ def testStartScheduledFlowsCreatesFlow(self): self.ScheduleFlow( client_id=client_id, creator=username, - flow_name=file.CollectSingleFile.__name__, - flow_args=rdf_file_finder.CollectSingleFileArgs(path="/foo"), - runner_args=rdf_flow_runner.FlowRunnerArgs(cpu_limit=60)) + flow_name=file.CollectFilesByKnownPath.__name__, + flow_args=rdf_file_finder.CollectFilesByKnownPathArgs(paths=["/foo"]), + runner_args=rdf_flow_runner.FlowRunnerArgs(cpu_limit=60), + ) flow.StartScheduledFlows(client_id, username) @@ -602,10 +651,13 @@ def testStartScheduledFlowsCreatesFlow(self): self.assertEqual(flows[0].client_id, client_id) self.assertEqual(flows[0].creator, username) - self.assertEqual(flows[0].flow_class_name, file.CollectSingleFile.__name__) - self.assertEqual(flows[0].args.path, "/foo") - self.assertEqual(flows[0].flow_state, - rdf_flow_objects.Flow.FlowState.RUNNING) + self.assertEqual( + flows[0].flow_class_name, file.CollectFilesByKnownPath.__name__ + ) + self.assertEqual(flows[0].args.paths, ["/foo"]) + self.assertEqual( + flows[0].flow_state, rdf_flow_objects.Flow.FlowState.RUNNING + ) self.assertEqual(flows[0].cpu_limit, 60) def testStartScheduledFlowsDeletesScheduledFlows(self): @@ -654,13 +706,16 @@ def testStartScheduledFlowsHandlesErrorInFlowConstructor(self): self.ScheduleFlow( client_id=client_id, creator=username, - flow_name=file.CollectSingleFile.__name__, - flow_args=rdf_file_finder.CollectSingleFileArgs(path="/foo"), - runner_args=rdf_flow_runner.FlowRunnerArgs(cpu_limit=60)) + flow_name=file.CollectFilesByKnownPath.__name__, + flow_args=rdf_file_finder.CollectFilesByKnownPathArgs(paths=["/foo"]), + runner_args=rdf_flow_runner.FlowRunnerArgs(cpu_limit=60), + ) with mock.patch.object( - file.CollectSingleFile, "__init__", - side_effect=ValueError("foobazzle")): + file.CollectFilesByKnownPath, + "__init__", + side_effect=ValueError("foobazzle"), + ): flow.StartScheduledFlows(client_id, username) self.assertEmpty(data_store.REL_DB.ReadAllFlowObjects(client_id)) @@ -676,14 +731,16 @@ def testStartScheduledFlowsHandlesErrorInFlowArgsValidation(self): self.ScheduleFlow( client_id=client_id, creator=username, - flow_name=file.CollectSingleFile.__name__, - flow_args=rdf_file_finder.CollectSingleFileArgs(path="/foo"), - runner_args=rdf_flow_runner.FlowRunnerArgs(cpu_limit=60)) + flow_name=file.CollectFilesByKnownPath.__name__, + flow_args=rdf_file_finder.CollectFilesByKnownPathArgs(paths=["/foo"]), + runner_args=rdf_flow_runner.FlowRunnerArgs(cpu_limit=60), + ) with mock.patch.object( - rdf_file_finder.CollectSingleFileArgs, + rdf_file_finder.CollectFilesByKnownPathArgs, "Validate", - side_effect=ValueError("foobazzle")): + side_effect=ValueError("foobazzle"), + ): flow.StartScheduledFlows(client_id, username) self.assertEmpty(data_store.REL_DB.ReadAllFlowObjects(client_id)) @@ -699,21 +756,24 @@ def testStartScheduledFlowsContinuesNextOnFailure(self): self.ScheduleFlow( client_id=client_id, creator=username, - flow_name=file.CollectSingleFile.__name__, - flow_args=rdf_file_finder.CollectSingleFileArgs(path="/foo"), - runner_args=rdf_flow_runner.FlowRunnerArgs(cpu_limit=60)) + flow_name=file.CollectFilesByKnownPath.__name__, + flow_args=rdf_file_finder.CollectFilesByKnownPathArgs(paths=["/foo"]), + runner_args=rdf_flow_runner.FlowRunnerArgs(cpu_limit=60), + ) self.ScheduleFlow( client_id=client_id, creator=username, - flow_name=file.CollectSingleFile.__name__, - flow_args=rdf_file_finder.CollectSingleFileArgs(path="/foo"), - runner_args=rdf_flow_runner.FlowRunnerArgs(cpu_limit=60)) + flow_name=file.CollectFilesByKnownPath.__name__, + flow_args=rdf_file_finder.CollectFilesByKnownPathArgs(paths=["/foo"]), + runner_args=rdf_flow_runner.FlowRunnerArgs(cpu_limit=60), + ) with mock.patch.object( - rdf_file_finder.CollectSingleFileArgs, + rdf_file_finder.CollectFilesByKnownPathArgs, "Validate", - side_effect=[ValueError("foobazzle"), mock.DEFAULT]): + side_effect=[ValueError("foobazzle"), mock.DEFAULT], + ): flow.StartScheduledFlows(client_id, username) self.assertLen(data_store.REL_DB.ReadAllFlowObjects(client_id), 1) diff --git a/grr/server/grr_response_server/flows/cron/system_test.py b/grr/server/grr_response_server/flows/cron/system_test.py index 4d653306c2..88e831fe2d 100644 --- a/grr/server/grr_response_server/flows/cron/system_test.py +++ b/grr/server/grr_response_server/flows/cron/system_test.py @@ -35,7 +35,6 @@ def setUp(self): self.SetupClientsWithIndices( range(20, 22), system="Darwin", - fleetspeak_enabled=True, ping=one_hour_ping) # These clients shouldn't be analyzed by any of the stats cronjobs. self.SetupClientsWithIndices( diff --git a/grr/server/grr_response_server/flows/file.py b/grr/server/grr_response_server/flows/file.py index 02e43b5f9a..c09e716b26 100644 --- a/grr/server/grr_response_server/flows/file.py +++ b/grr/server/grr_response_server/flows/file.py @@ -16,91 +16,6 @@ _MAX_FILE_SIZE = 1024 * 1024 * 1024 * 10 # 10 GiB. -# Although MultiGetFileLogic is a leaky, complex, and overall problematic Mixin -# it seems to be best choice to fetch the stat, hashes, and contents of a file. -# At the time of writing, none of the flows exposed all three to the caller in -# a sensible way. -class CollectSingleFile(transfer.MultiGetFileLogic, flow_base.FlowBase): - """Fetches contents of a single file from the specified absolute path.""" - friendly_name = "File content" - category = "/Filesystem/" - args_type = rdf_file_finder.CollectSingleFileArgs - result_types = (rdf_file_finder.CollectSingleFileResult,) - progress_type = rdf_file_finder.CollectSingleFileProgress - behaviours = flow_base.BEHAVIOUR_DEBUG - - def GetProgress(self) -> rdf_file_finder.CollectSingleFileProgress: - return self.state.progress - - def Start(self): - super().Start(file_size=self.args.max_size_bytes) - - self.state.progress = rdf_file_finder.CollectSingleFileProgress( - status=rdf_file_finder.CollectSingleFileProgress.Status.IN_PROGRESS) - - pathspec = rdf_paths.PathSpec.OS(path=self.args.path) - self.StartFileFetch(pathspec) - - def ReceiveFetchedFile(self, - stat_entry, - hash_obj, - request_data=None, - is_duplicate=False): - """See MultiGetFileLogic.""" - del request_data, is_duplicate # Unused. - - result = rdf_file_finder.CollectSingleFileResult( - stat=stat_entry, hash=hash_obj) - self.SendReply(result) - - self.state.progress.result = result - self.state.progress.status = ( - rdf_file_finder.CollectSingleFileProgress.Status.COLLECTED) - - def FileFetchFailed(self, - pathspec: rdf_paths.PathSpec, - request_data: Any = None, - status: Optional[rdf_flow_objects.FlowStatus] = None): - """See MultiGetFileLogic.""" - if (self.client_os == "Windows" and - pathspec.pathtype == rdf_paths.PathSpec.PathType.OS): - # Retry with raw filesystem access on Windows, - # the file might be locked for reads. - raw_pathspec = rdf_paths.PathSpec( - path=self.args.path, - pathtype=config.CONFIG["Server.raw_filesystem_access_pathtype"]) - self.StartFileFetch(raw_pathspec) - elif status is not None and status.error_message: - error_description = "{} when fetching {} with {}".format( - status.error_message, pathspec.path, pathspec.pathtype) - - # TODO: this is a really bad hack and should be fixed by - # passing the 'not found' status in a more structured way. - if "File not found" in status.error_message: - self.state.progress.status = rdf_file_finder.CollectSingleFileProgress.Status.NOT_FOUND - else: - self.state.progress.status = rdf_file_finder.CollectSingleFileProgress.Status.FAILED - self.state.progress.error_description = error_description - - raise flow_base.FlowError(error_description) - else: - error_description = ( - "File {} could not be fetched with {} due to an unknown error. " - "Check the flow logs.".format(pathspec.path, pathspec.pathtype)) - - self.state.progress.status = rdf_file_finder.CollectSingleFileProgress.Status.FAILED - self.state.progress.error_description = error_description - - raise flow_base.FlowError(error_description) - - @classmethod - def GetDefaultArgs(cls, username=None): - """See base class.""" - del username # Unused. - return rdf_file_finder.CollectSingleFileArgs( - path="", max_size_bytes="1 GiB") - - # Although MultiGetFileLogic is a leaky, complex, and overall problematic Mixin # it seems to be best choice to fetch the stat, hashes, and contents of a file. # At the time of writing, none of the flows exposed all three to the caller in @@ -291,61 +206,27 @@ def Start(self): num_failed=0, ) - conditions = [] - - if self.args.HasField("modification_time"): - conditions.append( - rdf_file_finder.FileFinderCondition( - condition_type=rdf_file_finder.FileFinderCondition.Type - .MODIFICATION_TIME, - modification_time=self.args.modification_time, - )) - - if self.args.HasField("access_time"): - conditions.append( - rdf_file_finder.FileFinderCondition( - condition_type=rdf_file_finder.FileFinderCondition.Type - .ACCESS_TIME, - access_time=self.args.access_time, - )) - - if self.args.HasField("inode_change_time"): - conditions.append( - rdf_file_finder.FileFinderCondition( - condition_type=rdf_file_finder.FileFinderCondition.Type - .INODE_CHANGE_TIME, - inode_change_time=self.args.inode_change_time, - )) - - if self.args.HasField("size"): - conditions.append( - rdf_file_finder.FileFinderCondition( - condition_type=rdf_file_finder.FileFinderCondition.Type.SIZE, - size=self.args.size, - )) - - if self.args.HasField("ext_flags"): - conditions.append( - rdf_file_finder.FileFinderCondition( - condition_type=rdf_file_finder.FileFinderCondition.Type.EXT_FLAGS, - ext_flags=self.args.ext_flags, - )) - - if self.args.HasField("contents_regex_match"): - conditions.append( - rdf_file_finder.FileFinderCondition( - condition_type=rdf_file_finder.FileFinderCondition.Type - .CONTENTS_REGEX_MATCH, - contents_regex_match=self.args.contents_regex_match, - )) - - if self.args.HasField("contents_literal_match"): - conditions.append( - rdf_file_finder.FileFinderCondition( - condition_type=rdf_file_finder.FileFinderCondition.Type - .CONTENTS_LITERAL_MATCH, - contents_literal_match=self.args.contents_literal_match, - )) + conditions = BuildClientFileFinderConditions( + modification_time=self.args.modification_time + if self.args.HasField("modification_time") + else None, + access_time=self.args.access_time + if self.args.HasField("access_time") + else None, + inode_change_time=self.args.inode_change_time + if self.args.HasField("inode_change_time") + else None, + size=self.args.size if self.args.HasField("size") else None, + ext_flags=self.args.ext_flags + if self.args.HasField("ext_flags") + else None, + contents_regex_match=self.args.contents_regex_match + if self.args.HasField("contents_regex_match") + else None, + contents_literal_match=self.args.contents_literal_match + if self.args.HasField("contents_literal_match") + else None, + ) file_finder_args = rdf_file_finder.FileFinderArgs( paths=self.args.path_expressions, @@ -364,60 +245,379 @@ def ProcessFiles(self, responses): for response in responses: pathspec = response.stat_entry.pathspec - self.StartFileFetch(pathspec, request_data=dict(original_result=response)) + self.StartFileFetch( + pathspec, request_data=dict(original_pathspec=pathspec) + ) self.state.progress.num_found += 1 self.state.progress.num_in_progress += 1 - def ReceiveFetchedFile(self, - stat_entry, - hash_obj, - request_data=None, - is_duplicate=False): + def ReceiveFetchedFile( + self, stat_entry, hash_obj, request_data=None, is_duplicate=False + ): """See MultiGetFileLogic.""" del request_data, is_duplicate # Unused. + self.state.progress.num_in_progress = max( + 0, self.state.progress.num_in_progress - 1 + ) + self.state.progress.num_collected += 1 + result = rdf_file_finder.CollectMultipleFilesResult( stat=stat_entry, hash=hash_obj, - status=rdf_file_finder.CollectMultipleFilesResult.Status.COLLECTED) + status=rdf_file_finder.CollectMultipleFilesResult.Status.COLLECTED, + ) self.SendReply(result) - self.state.progress.num_in_progress = max( - 0, self.state.progress.num_in_progress - 1) - self.state.progress.num_collected += 1 - - def FileFetchFailed(self, - pathspec: rdf_paths.PathSpec, - request_data: Any = None, - status: Optional[rdf_flow_objects.FlowStatus] = None): + def FileFetchFailed( + self, + pathspec: rdf_paths.PathSpec, + request_data: Any = None, + status: Optional[rdf_flow_objects.FlowStatus] = None, + ): """See MultiGetFileLogic.""" - original_result = request_data["original_result"] - - if (self.client_os == "Windows" and - pathspec.pathtype == rdf_paths.PathSpec.PathType.OS): + original_pathspec = pathspec + if request_data is not None and request_data["original_pathspec"]: + original_pathspec = request_data["original_pathspec"] + + if ( + self.client_os == "Windows" + and pathspec.pathtype == rdf_paths.PathSpec.PathType.OS + ): # Retry with raw filesystem access on Windows, # the file might be locked for reads. raw_pathspec = rdf_paths.PathSpec( - path=self.args.path, - pathtype=config.CONFIG["Server.raw_filesystem_access_pathtype"]) - self.StartFileFetch(raw_pathspec) + path=original_pathspec.path, + pathtype=config.CONFIG["Server.raw_filesystem_access_pathtype"], + ) + self.StartFileFetch( + raw_pathspec, request_data=dict(original_pathspec=raw_pathspec) + ) self.state.progress.num_raw_fs_access_retries += 1 else: if status is not None and status.error_message: error_description = "{} when fetching {} with {}".format( - status.error_message, pathspec.path, pathspec.pathtype) + status.error_message, pathspec.path, pathspec.pathtype + ) else: error_description = ( "File {} could not be fetched with {} due to an unknown error. " - "Check the flow logs.".format(pathspec.path, pathspec.pathtype)) + "Check the flow logs.".format(pathspec.path, pathspec.pathtype) + ) + + self.state.progress.num_in_progress = max( + 0, self.state.progress.num_in_progress - 1 + ) + self.state.progress.num_failed += 1 result = rdf_file_finder.CollectMultipleFilesResult( - stat=original_result.stat_entry, + stat=rdf_client_fs.StatEntry(pathspec=original_pathspec), error=error_description, status=rdf_file_finder.CollectMultipleFilesResult.Status.FAILED, ) self.SendReply(result) + +class StatMultipleFiles(flow_base.FlowBase): + """Fetches file stats by searching for path expressions.""" + + friendly_name = "Collect file stats" + category = "/Filesystem/" + behaviours = flow_base.BEHAVIOUR_BASIC + + args_type = rdf_file_finder.StatMultipleFilesArgs + result_types = (rdf_client_fs.StatEntry,) + + def Start(self): + conditions = BuildClientFileFinderConditions( + modification_time=self.args.modification_time + if self.args.HasField("modification_time") + else None, + access_time=self.args.access_time + if self.args.HasField("access_time") + else None, + inode_change_time=self.args.inode_change_time + if self.args.HasField("inode_change_time") + else None, + size=self.args.size if self.args.HasField("size") else None, + ext_flags=self.args.ext_flags + if self.args.HasField("ext_flags") + else None, + contents_regex_match=self.args.contents_regex_match + if self.args.HasField("contents_regex_match") + else None, + contents_literal_match=self.args.contents_literal_match + if self.args.HasField("contents_literal_match") + else None, + ) + + file_finder_args = rdf_file_finder.FileFinderArgs( + paths=self.args.path_expressions, + pathtype=rdf_paths.PathSpec.PathType.OS, + conditions=conditions, + action=rdf_file_finder.FileFinderAction.Stat(), + ) + + self.CallFlow( + file_finder.ClientFileFinder.__name__, + flow_args=file_finder_args, + next_state=self.ProcessResponses.__name__, + ) + + def ProcessResponses(self, responses): + if not responses.success: + raise flow_base.FlowError(responses.status.error_message) + + for response in responses: + result = rdf_client_fs.StatEntry(pathspec=response.stat_entry.pathspec) + + self.SendReply(result) + + +class HashMultipleFiles(transfer.MultiGetFileLogic, flow_base.FlowBase): + """Fetches file hashes and stats by searching for path expressions.""" + + friendly_name = "Collect file hashes" + category = "/Filesystem/" + behaviours = flow_base.BEHAVIOUR_BASIC + + args_type = rdf_file_finder.HashMultipleFilesArgs + result_types = (rdf_file_finder.CollectMultipleFilesResult,) + progress_type = rdf_file_finder.HashMultipleFilesProgress + + MAX_FILE_SIZE = 1024 * 1024 * 1024 * 10 # 10GiB + + def GetProgress(self) -> rdf_file_finder.HashMultipleFilesProgress: + return self.state.progress + + def Start(self): + """See base class.""" + super().Start(file_size=self.MAX_FILE_SIZE) + + self.state.progress = rdf_file_finder.HashMultipleFilesProgress( + num_found=0, + num_in_progress=0, + num_raw_fs_access_retries=0, + num_hashed=0, + num_failed=0, + ) + + # Set the collection level for MultiGetFileLogic mixin, as the default + # one is collecting the file contents + self.state.stop_at_hash = True + + conditions = BuildClientFileFinderConditions( + modification_time=self.args.modification_time + if self.args.HasField("modification_time") + else None, + access_time=self.args.access_time + if self.args.HasField("access_time") + else None, + inode_change_time=self.args.inode_change_time + if self.args.HasField("inode_change_time") + else None, + size=self.args.size if self.args.HasField("size") else None, + ext_flags=self.args.ext_flags + if self.args.HasField("ext_flags") + else None, + contents_regex_match=self.args.contents_regex_match + if self.args.HasField("contents_regex_match") + else None, + contents_literal_match=self.args.contents_literal_match + if self.args.HasField("contents_literal_match") + else None, + ) + + file_finder_args = rdf_file_finder.FileFinderArgs( + paths=self.args.path_expressions, + pathtype=rdf_paths.PathSpec.PathType.OS, + conditions=conditions, + action=rdf_file_finder.FileFinderAction.Hash(), + ) + + self.CallFlow( + file_finder.ClientFileFinder.__name__, + flow_args=file_finder_args, + next_state=self.ProcessResponses.__name__, + ) + + def ProcessResponses(self, responses): + if not responses.success: + raise flow_base.FlowError(responses.status.error_message) + + for response in responses: + pathspec = response.stat_entry.pathspec + self.StartFileFetch( + pathspec, request_data=dict(original_pathspec=pathspec) + ) + self.state.progress.num_found += 1 + self.state.progress.num_in_progress += 1 + + def FileFetchFailed( + self, + pathspec: rdf_paths.PathSpec, + request_data: Any = None, + status: Optional[rdf_flow_objects.FlowStatus] = None, + ): + """See MultiGetFileLogic.""" + original_pathspec = pathspec + if request_data is not None and request_data["original_pathspec"]: + original_pathspec = request_data["original_pathspec"] + + if ( + self.client_os == "Windows" + and pathspec.pathtype == rdf_paths.PathSpec.PathType.OS + ): + # Retry with raw filesystem access on Windows, + # the file might be locked for reads. + raw_pathspec = rdf_paths.PathSpec( + path=original_pathspec.path, + pathtype=config.CONFIG["Server.raw_filesystem_access_pathtype"], + ) + self.StartFileFetch( + raw_pathspec, request_data=dict(original_pathspec=raw_pathspec) + ) + + self.state.progress.num_raw_fs_access_retries += 1 + else: + if status is not None and status.error_message: + error_description = "{} when fetching {} with {}".format( + status.error_message, pathspec.path, pathspec.pathtype + ) + else: + error_description = ( + "File {} could not be fetched with {} due to an unknown error. " + "Check the flow logs.".format(pathspec.path, pathspec.pathtype) + ) + self.state.progress.num_in_progress = max( - 0, self.state.progress.num_in_progress - 1) + 0, self.state.progress.num_in_progress - 1 + ) + self.state.progress.num_failed += 1 + + result = rdf_file_finder.CollectMultipleFilesResult( + stat=rdf_client_fs.StatEntry(pathspec=original_pathspec), + error=error_description, + status=rdf_file_finder.CollectMultipleFilesResult.Status.FAILED, + ) + self.SendReply(result) + + def ReceiveFetchedFileHash( + self, + stat_entry: rdf_client_fs.StatEntry, + file_hash: rdf_crypto.Hash, + request_data: Optional[Mapping[str, Any]] = None, + ): + """This method will be called for each new file hash successfully fetched. + + Args: + stat_entry: rdf_client_fs.StatEntry object describing the file. + file_hash: rdf_crypto.Hash object with file hashes. + request_data: Arbitrary dictionary that was passed to the corresponding + StartFileFetch call. + """ + del request_data # Unused. + + self.state.progress.num_in_progress -= 1 + self.state.progress.num_hashed += 1 + + result = rdf_file_finder.CollectMultipleFilesResult( + stat=stat_entry, + hash=file_hash, + status=rdf_file_finder.CollectMultipleFilesResult.Status.COLLECTED, + ) + + self.SendReply(result) + + +def BuildClientFileFinderConditions( + modification_time: Optional[ + rdf_file_finder.FileFinderModificationTimeCondition + ] = None, + access_time: Optional[rdf_file_finder.FileFinderAccessTimeCondition] = None, + inode_change_time: Optional[ + rdf_file_finder.FileFinderInodeChangeTimeCondition + ] = None, + size: Optional[rdf_file_finder.FileFinderSizeCondition] = None, + ext_flags: Optional[rdf_file_finder.FileFinderExtFlagsCondition] = None, + contents_regex_match: Optional[ + rdf_file_finder.FileFinderContentsRegexMatchCondition + ] = None, + contents_literal_match: Optional[ + rdf_file_finder.FileFinderContentsLiteralMatchCondition + ] = None, +) -> list[rdf_file_finder.FileFinderCondition]: + """Constructs the list of conditions to be applied to ClientFileFinder flow. + + Args: + modification_time: Min/max last modification time of the file(s). + access_time: Min/max last access time of the file(s). + inode_change_time: Min/max last inode time of the file(s). + size: Min/max file size. + ext_flags: Linux and/or macOS file flags. + contents_regex_match: regex rule to match in the file contents. + contents_literal_match: string literal to match in the file contents. + + Returns: + List of file conditions for ClientFileFinder flow. + """ + conditions = [] + + if modification_time is not None: + conditions.append( + rdf_file_finder.FileFinderCondition( + condition_type=rdf_file_finder.FileFinderCondition.Type.MODIFICATION_TIME, + modification_time=modification_time, + ) + ) + + if access_time is not None: + conditions.append( + rdf_file_finder.FileFinderCondition( + condition_type=rdf_file_finder.FileFinderCondition.Type.ACCESS_TIME, + access_time=access_time, + ) + ) + + if inode_change_time is not None: + conditions.append( + rdf_file_finder.FileFinderCondition( + condition_type=rdf_file_finder.FileFinderCondition.Type.INODE_CHANGE_TIME, + inode_change_time=inode_change_time, + ) + ) + + if size is not None: + conditions.append( + rdf_file_finder.FileFinderCondition( + condition_type=rdf_file_finder.FileFinderCondition.Type.SIZE, + size=size, + ) + ) + + if ext_flags is not None: + conditions.append( + rdf_file_finder.FileFinderCondition( + condition_type=rdf_file_finder.FileFinderCondition.Type.EXT_FLAGS, + ext_flags=ext_flags, + ) + ) + + if contents_regex_match is not None: + conditions.append( + rdf_file_finder.FileFinderCondition( + condition_type=rdf_file_finder.FileFinderCondition.Type.CONTENTS_REGEX_MATCH, + contents_regex_match=contents_regex_match, + ) + ) + + if contents_literal_match is not None: + conditions.append( + rdf_file_finder.FileFinderCondition( + condition_type=rdf_file_finder.FileFinderCondition.Type.CONTENTS_LITERAL_MATCH, + contents_literal_match=contents_literal_match, + ) + ) + + return conditions diff --git a/grr/server/grr_response_server/flows/file_test.py b/grr/server/grr_response_server/flows/file_test.py index fcb864f3e6..028375bdc9 100644 --- a/grr/server/grr_response_server/flows/file_test.py +++ b/grr/server/grr_response_server/flows/file_test.py @@ -71,128 +71,6 @@ def Read(self, length=None): }) -class TestCollectSingleFile(flow_test_lib.FlowTestsBaseclass): - - def setUp(self): - super().setUp() - self.client_id = self.SetupClient(0) - self.client_mock = action_mocks.FileFinderClientMockWithTimestamps() - - stack = contextlib.ExitStack() - self.files = stack.enter_context(SetUpTestFiles()) - self.addCleanup(stack.close) - - def testCollectSingleFileReturnsFile(self): - flow_id = flow_test_lib.TestFlowHelper( - file.CollectSingleFile.__name__, - self.client_mock, - client_id=self.client_id, - path=self.files["bar"].path, - creator=self.test_username) - - results = flow_test_lib.GetFlowResults(self.client_id, flow_id) - self.assertLen(results, 1) - self.assertEqual(results[0].stat.pathspec.path, self.files["bar"].path) - self.assertEqual(results[0].stat.pathspec.pathtype, - rdf_paths.PathSpec.PathType.OS) - self.assertEqual(str(results[0].hash.sha1), self.files["bar"].sha1) - - def testFileNotFoundRaisesError(self): - with self.assertRaisesRegex(RuntimeError, "File not found"): - flow_test_lib.TestFlowHelper( - file.CollectSingleFile.__name__, - self.client_mock, - client_id=self.client_id, - path="/nonexistent", - creator=self.test_username) - - def testFetchIsRetriedWithRawOnWindows(self): - with mock.patch.object(flow_base.FlowBase, "client_os", "Windows"): - with _PatchVfs(): - flow_id = flow_test_lib.TestFlowHelper( - file.CollectSingleFile.__name__, - self.client_mock, - client_id=self.client_id, - path=self.files["bar"].path, - creator=self.test_username) - - results = flow_test_lib.GetFlowResults(self.client_id, flow_id) - - self.assertLen(results, 1) - self.assertEqual(results[0].stat.pathspec.path, self.files["bar"].path) - self.assertEqual(results[0].stat.pathspec.pathtype, - config.CONFIG["Server.raw_filesystem_access_pathtype"]) - self.assertEqual(str(results[0].hash.sha1), self.files["bar"].sha1) - - def testRaisesAfterRetryOnFailedFetchOnWindows(self): - with mock.patch.object(flow_base.FlowBase, "client_os", "Windows"): - with mock.patch.object(vfs, "VFSOpen", side_effect=IOError("mock err")): - with self.assertRaisesRegex(RuntimeError, r"mock err.*bar.*") as e: - flow_test_lib.TestFlowHelper( - file.CollectSingleFile.__name__, - self.client_mock, - client_id=self.client_id, - path=self.files["bar"].path, - creator=self.test_username) - self.assertIn( - str(config.CONFIG["Server.raw_filesystem_access_pathtype"]), - str(e.exception)) - - def testCorrectlyReportsProgressInFlight(self): - flow_id = flow_test_lib.StartFlow( - file.CollectSingleFile, client_id=self.client_id, path="/some/path") - progress = flow_test_lib.GetFlowProgress(self.client_id, flow_id) - self.assertEqual( - progress.status, - rdf_file_finder.CollectSingleFileProgress.Status.IN_PROGRESS) - - def testProgressContainsResultOnSuccess(self): - flow_id = flow_test_lib.StartAndRunFlow( - file.CollectSingleFile, - self.client_mock, - client_id=self.client_id, - path=self.files["bar"].path) - - progress = flow_test_lib.GetFlowProgress(self.client_id, flow_id) - - self.assertEqual(progress.status, - rdf_file_finder.CollectSingleFileProgress.Status.COLLECTED) - self.assertEqual(progress.result.stat.pathspec.path, self.files["bar"].path) - self.assertEqual(progress.result.stat.pathspec.pathtype, - rdf_paths.PathSpec.PathType.OS) - self.assertEqual(str(progress.result.hash.sha1), self.files["bar"].sha1) - - def testProgressCorrectlyIndicatesNotFoundStatus(self): - flow_id = flow_test_lib.StartAndRunFlow( - file.CollectSingleFile, - self.client_mock, - client_id=self.client_id, - check_flow_errors=False, - path="/nonexistent") - - progress = flow_test_lib.GetFlowProgress(self.client_id, flow_id) - self.assertEqual(progress.status, - rdf_file_finder.CollectSingleFileProgress.Status.NOT_FOUND) - - def testProgressCorrectlyIndicatesErrorStatus(self): - with mock.patch.object(flow_base.FlowBase, "client_os", "Windows"): - with mock.patch.object(vfs, "VFSOpen", side_effect=IOError("mock err")): - flow_id = flow_test_lib.StartAndRunFlow( - file.CollectSingleFile, - self.client_mock, - client_id=self.client_id, - check_flow_errors=False, - path="/nonexistent") - - progress = flow_test_lib.GetFlowProgress(self.client_id, flow_id) - self.assertEqual(progress.status, - rdf_file_finder.CollectSingleFileProgress.Status.FAILED) - self.assertEqual( - progress.error_description, - f"mock err when fetching /nonexistent with {config.CONFIG['Server.raw_filesystem_access_pathtype']}" - ) - - class TestCollectFilesByKnownPath(flow_test_lib.FlowTestsBaseclass): def setUp(self): @@ -601,6 +479,46 @@ def testCollectMultipleFilesReturnsFiles(self): self.assertEqual(f.stat.pathspec.path, self.files["baz"].path) self.assertEqual(str(f.hash.sha1), self.files["baz"].sha1) + def testFailsAfterRetryOnFailedFetchOnWindows(self): + temp_bar_file = self.create_tempfile() + temp_bar_file.write_bytes(b"bar") + file_bar_path = temp_bar_file.full_path + + with mock.patch.object(flow_base.FlowBase, "client_os", "Windows"): + with mock.patch.object(vfs, "VFSOpen", side_effect=IOError("mock err")): + flow_id = flow_test_lib.TestFlowHelper( + file.CollectMultipleFiles.__name__, + self.client_mock, + client_id=self.client_id, + path_expressions=[file_bar_path], + creator=self.test_username, + ) + + progress = flow_test_lib.GetFlowProgress(self.client_id, flow_id) + self.assertEqual( + rdf_file_finder.CollectMultipleFilesProgress( + num_found=1, + num_in_progress=0, + num_raw_fs_access_retries=1, + num_collected=0, + num_failed=1, + ), + progress, + ) + + results = flow_test_lib.GetFlowResults(self.client_id, flow_id) + self.assertLen(results, 1) + self.assertEqual(results[0].stat.pathspec.path, file_bar_path) + self.assertEqual( + results[0].stat.pathspec.pathtype, + config.CONFIG["Server.raw_filesystem_access_pathtype"], + ) + self.assertIsNone(results[0].hash.sha1) + self.assertEqual( + results[0].status, + rdf_file_finder.CollectMultipleFilesResult.Status.FAILED, + ) + def testCorrectlyReportProgressForSuccessfullyCollectedFiles(self): path = os.path.join(self.fixture_path, "b*") @@ -718,6 +636,527 @@ def _GetCondition(condition_type): contents_literal_match) +class TestStatMultipleFiles(flow_test_lib.FlowTestsBaseclass): + + def setUp(self): + super().setUp() + self.client_id = self.SetupClient(0) + self.client_mock = action_mocks.CollectMultipleFilesClientMock() + + def testReturnsSingleFileStat(self): + temp_bar_file = self.create_tempfile() + temp_bar_file.write_bytes(b"bar") + file_bar_path = temp_bar_file.full_path + + flow_id = flow_test_lib.TestFlowHelper( + file.StatMultipleFiles.__name__, + self.client_mock, + client_id=self.client_id, + path_expressions=[file_bar_path], + creator=self.test_username, + ) + + results = flow_test_lib.GetFlowResults(self.client_id, flow_id) + + self.assertLen(results, 1) + self.assertEqual( + results[0].pathspec.pathtype, + rdf_paths.PathSpec.PathType.OS, + ) + self.assertEqual(file_bar_path, results[0].pathspec.path) + + def testReturnsMultipleFileStats(self): + temp_bar_file = self.create_tempfile() + temp_bar_file.write_bytes(b"bar") + file_bar_path = temp_bar_file.full_path + + temp_foo_file = self.create_tempfile() + temp_foo_file.write_bytes(b"foo") + file_foo_path = temp_foo_file.full_path + + flow_id = flow_test_lib.TestFlowHelper( + file.StatMultipleFiles.__name__, + self.client_mock, + client_id=self.client_id, + path_expressions=[file_bar_path, file_foo_path], + creator=self.test_username, + ) + + results = flow_test_lib.GetFlowResults(self.client_id, flow_id) + + self.assertLen(results, 2) + + self.assertEqual(results[0].pathspec.path, file_bar_path) + self.assertEqual( + results[0].pathspec.pathtype, + rdf_paths.PathSpec.PathType.OS, + ) + + self.assertEqual(results[1].pathspec.path, file_foo_path) + self.assertEqual( + results[1].pathspec.pathtype, + rdf_paths.PathSpec.PathType.OS, + ) + + def testFileNotFound(self): + temp_dir = self.create_tempdir() + non_existent_file_path = os.path.join(temp_dir.full_path, "/non_existent") + + flow_id = flow_test_lib.TestFlowHelper( + file.StatMultipleFiles.__name__, + self.client_mock, + client_id=self.client_id, + path_expressions=[non_existent_file_path], + creator=self.test_username, + ) + + results = flow_test_lib.GetFlowResults(self.client_id, flow_id) + self.assertEmpty(results) + + def testPassesNoConditionsToClientFileFinderWhenNoConditionsSpecified(self): + flow_id = flow_test_lib.StartFlow( + file.StatMultipleFiles, + client_id=self.client_id, + path_expressions=["/some/path"], + ) + + children = data_store.REL_DB.ReadChildFlowObjects(self.client_id, flow_id) + self.assertLen(children, 1) + + child = children[0] + self.assertEqual( + child.flow_class_name, file_finder.ClientFileFinder.__name__ + ) + self.assertEmpty(child.args.conditions) + + def testPassesAllConditionsToClientFileFinderWhenAllConditionsSpecified(self): + modification_time = rdf_file_finder.FileFinderModificationTimeCondition( + min_last_modified_time=rdfvalue.RDFDatetime.Now(), + ) + + access_time = rdf_file_finder.FileFinderAccessTimeCondition( + min_last_access_time=rdfvalue.RDFDatetime.Now(), + ) + + inode_change_time = rdf_file_finder.FileFinderInodeChangeTimeCondition( + min_last_inode_change_time=rdfvalue.RDFDatetime.Now(), + ) + + size = rdf_file_finder.FileFinderSizeCondition( + min_file_size=42, + ) + + ext_flags = rdf_file_finder.FileFinderExtFlagsCondition( + linux_bits_set=42, + ) + + contents_regex_match = ( + rdf_file_finder.FileFinderContentsRegexMatchCondition( + regex=b"foo", + ) + ) + + contents_literal_match = ( + rdf_file_finder.FileFinderContentsLiteralMatchCondition( + literal=b"bar", + ) + ) + + flow_id = flow_test_lib.StartFlow( + file.StatMultipleFiles, + client_id=self.client_id, + path_expressions=["/some/path"], + modification_time=modification_time, + access_time=access_time, + inode_change_time=inode_change_time, + size=size, + ext_flags=ext_flags, + contents_regex_match=contents_regex_match, + contents_literal_match=contents_literal_match, + ) + + children = data_store.REL_DB.ReadChildFlowObjects(self.client_id, flow_id) + self.assertLen(children, 1) + + child = children[0] + self.assertEqual( + child.flow_class_name, file_finder.ClientFileFinder.__name__ + ) + # We expect 7 condition-attributes to be converted + # to 7 FileFinderConditions. + self.assertLen(child.args.conditions, 7) + + def _GetCondition(condition_type): + for c in child.args.conditions: + if c.condition_type == condition_type: + return c.UnionCast() + + raise RuntimeError(f"Condition of type {condition_type} not found.") + + self.assertEqual( + _GetCondition( + rdf_file_finder.FileFinderCondition.Type.MODIFICATION_TIME + ), + modification_time, + ) + + self.assertEqual( + _GetCondition(rdf_file_finder.FileFinderCondition.Type.ACCESS_TIME), + access_time, + ) + + self.assertEqual( + _GetCondition( + rdf_file_finder.FileFinderCondition.Type.INODE_CHANGE_TIME + ), + inode_change_time, + ) + + self.assertEqual( + _GetCondition(rdf_file_finder.FileFinderCondition.Type.SIZE), size + ) + + self.assertEqual( + _GetCondition(rdf_file_finder.FileFinderCondition.Type.EXT_FLAGS), + ext_flags, + ) + + self.assertEqual( + _GetCondition( + rdf_file_finder.FileFinderCondition.Type.CONTENTS_REGEX_MATCH + ), + contents_regex_match, + ) + + self.assertEqual( + _GetCondition( + rdf_file_finder.FileFinderCondition.Type.CONTENTS_LITERAL_MATCH + ), + contents_literal_match, + ) + + +class TestHashMultipleFiles(flow_test_lib.FlowTestsBaseclass): + + def setUp(self): + super().setUp() + self.client_id = self.SetupClient(0) + self.client_mock = action_mocks.CollectMultipleFilesClientMock() + + def testReturnsSingleFileHash(self): + temp_bar_file = self.create_tempfile() + temp_bar_file.write_bytes(b"bar") + file_bar_path = temp_bar_file.full_path + file_bar_hash = hashlib.sha1(b"bar").hexdigest() + + flow_id = flow_test_lib.TestFlowHelper( + file.HashMultipleFiles.__name__, + self.client_mock, + client_id=self.client_id, + path_expressions=[file_bar_path], + creator=self.test_username, + ) + + progress = flow_test_lib.GetFlowProgress(self.client_id, flow_id) + self.assertEqual( + rdf_file_finder.HashMultipleFilesProgress( + num_found=1, + num_in_progress=0, + num_raw_fs_access_retries=0, + num_hashed=1, + num_failed=0, + ), + progress, + ) + + results = flow_test_lib.GetFlowResults(self.client_id, flow_id) + + self.assertLen(results, 1) + self.assertEqual( + results[0].stat.pathspec.pathtype, + rdf_paths.PathSpec.PathType.OS, + ) + self.assertEqual(file_bar_path, results[0].stat.pathspec.path) + self.assertEqual( + results[0].status, + rdf_file_finder.CollectMultipleFilesResult.Status.COLLECTED, + ) + self.assertEqual(results[0].hash.sha1, file_bar_hash) + + def testReturnsMultipleFileHashes(self): + temp_bar_file = self.create_tempfile() + temp_bar_file.write_bytes(b"bar") + file_bar_path = temp_bar_file.full_path + file_bar_hash = hashlib.sha1(b"bar").hexdigest() + + temp_foo_file = self.create_tempfile() + temp_foo_file.write_bytes(b"foo") + file_foo_path = temp_foo_file.full_path + file_foo_hash = hashlib.sha1(b"foo").hexdigest() + + flow_id = flow_test_lib.TestFlowHelper( + file.HashMultipleFiles.__name__, + self.client_mock, + client_id=self.client_id, + path_expressions=[file_bar_path, file_foo_path], + creator=self.test_username, + ) + + progress = flow_test_lib.GetFlowProgress(self.client_id, flow_id) + self.assertEqual( + rdf_file_finder.HashMultipleFilesProgress( + num_found=2, + num_in_progress=0, + num_raw_fs_access_retries=0, + num_hashed=2, + num_failed=0, + ), + progress, + ) + + results = flow_test_lib.GetFlowResults(self.client_id, flow_id) + + self.assertLen(results, 2) + self.assertEqual(results[0].stat.pathspec.path, file_bar_path) + self.assertEqual( + results[0].stat.pathspec.pathtype, rdf_paths.PathSpec.PathType.OS + ) + self.assertEqual( + results[0].status, + rdf_file_finder.CollectFilesByKnownPathResult.Status.COLLECTED, + ) + self.assertEqual(results[0].hash.sha1, file_bar_hash) + + self.assertEqual(results[1].stat.pathspec.path, file_foo_path) + self.assertEqual( + results[1].stat.pathspec.pathtype, rdf_paths.PathSpec.PathType.OS + ) + self.assertEqual( + results[1].status, + rdf_file_finder.CollectFilesByKnownPathResult.Status.COLLECTED, + ) + self.assertEqual(results[1].hash.sha1, file_foo_hash) + + def testFileNotFound(self): + temp_dir = self.create_tempdir() + non_existent_file_path = os.path.join(temp_dir.full_path, "/non_existent") + + flow_id = flow_test_lib.TestFlowHelper( + file.HashMultipleFiles.__name__, + self.client_mock, + client_id=self.client_id, + path_expressions=[non_existent_file_path], + creator=self.test_username, + ) + + progress = flow_test_lib.GetFlowProgress(self.client_id, flow_id) + self.assertEqual( + rdf_file_finder.HashMultipleFilesProgress( + num_found=0, + num_in_progress=0, + num_raw_fs_access_retries=0, + num_hashed=0, + num_failed=0, + ), + progress, + ) + + results = flow_test_lib.GetFlowResults(self.client_id, flow_id) + self.assertEmpty(results) + + def testFailsAfterRetryOnFailedFetchOnWindows(self): + temp_bar_file = self.create_tempfile() + temp_bar_file.write_bytes(b"bar") + file_bar_path = temp_bar_file.full_path + + with mock.patch.object(flow_base.FlowBase, "client_os", "Windows"): + with mock.patch.object(vfs, "VFSOpen", side_effect=IOError("mock err")): + flow_id = flow_test_lib.TestFlowHelper( + file.HashMultipleFiles.__name__, + self.client_mock, + client_id=self.client_id, + path_expressions=[file_bar_path], + creator=self.test_username, + ) + + progress = flow_test_lib.GetFlowProgress(self.client_id, flow_id) + + self.assertEqual( + rdf_file_finder.HashMultipleFilesProgress( + num_found=1, + num_in_progress=0, + num_raw_fs_access_retries=1, + num_hashed=0, + num_failed=1, + ), + progress, + ) + + results = flow_test_lib.GetFlowResults(self.client_id, flow_id) + self.assertLen(results, 1) + self.assertEqual(results[0].stat.pathspec.path, file_bar_path) + self.assertEqual( + results[0].stat.pathspec.pathtype, + config.CONFIG["Server.raw_filesystem_access_pathtype"], + ) + self.assertEqual( + results[0].status, + rdf_file_finder.CollectMultipleFilesResult.Status.FAILED, + ) + self.assertIsNone(results[0].hash.sha1) + + def testCorrectlyReportProgressForSuccessfullyCollectedFileHashes(self): + temp_bar_file = self.create_tempfile() + temp_bar_file.write_bytes(b"bar") + file_bar_path = temp_bar_file.full_path + + temp_foo_file = self.create_tempfile() + temp_foo_file.write_bytes(b"foo") + file_foo_path = temp_foo_file.full_path + + flow_id = flow_test_lib.TestFlowHelper( + file.HashMultipleFiles.__name__, + self.client_mock, + client_id=self.client_id, + path_expressions=[file_bar_path, file_foo_path], + creator=self.test_username, + ) + + progress = flow_test_lib.GetFlowProgress(self.client_id, flow_id) + self.assertEqual( + rdf_file_finder.HashMultipleFilesProgress( + num_hashed=2, + num_failed=0, + num_found=2, + num_in_progress=0, + num_raw_fs_access_retries=0, + ), + progress, + ) + + def testPassesNoConditionsToClientFileFinderWhenNoConditionsSpecified(self): + flow_id = flow_test_lib.StartFlow( + file.HashMultipleFiles, + client_id=self.client_id, + path_expressions=["/some/path"], + ) + + children = data_store.REL_DB.ReadChildFlowObjects(self.client_id, flow_id) + self.assertLen(children, 1) + + child = children[0] + self.assertEqual( + child.flow_class_name, file_finder.ClientFileFinder.__name__ + ) + self.assertEmpty(child.args.conditions) + + def testPassesAllConditionsToClientFileFinderWhenAllConditionsSpecified(self): + modification_time = rdf_file_finder.FileFinderModificationTimeCondition( + min_last_modified_time=rdfvalue.RDFDatetime.Now(), + ) + + access_time = rdf_file_finder.FileFinderAccessTimeCondition( + min_last_access_time=rdfvalue.RDFDatetime.Now(), + ) + + inode_change_time = rdf_file_finder.FileFinderInodeChangeTimeCondition( + min_last_inode_change_time=rdfvalue.RDFDatetime.Now(), + ) + + size = rdf_file_finder.FileFinderSizeCondition( + min_file_size=42, + ) + + ext_flags = rdf_file_finder.FileFinderExtFlagsCondition( + linux_bits_set=42, + ) + + contents_regex_match = ( + rdf_file_finder.FileFinderContentsRegexMatchCondition( + regex=b"foo", + ) + ) + + contents_literal_match = ( + rdf_file_finder.FileFinderContentsLiteralMatchCondition( + literal=b"bar", + ) + ) + + flow_id = flow_test_lib.StartFlow( + file.HashMultipleFiles, + client_id=self.client_id, + path_expressions=["/some/path"], + modification_time=modification_time, + access_time=access_time, + inode_change_time=inode_change_time, + size=size, + ext_flags=ext_flags, + contents_regex_match=contents_regex_match, + contents_literal_match=contents_literal_match, + ) + + children = data_store.REL_DB.ReadChildFlowObjects(self.client_id, flow_id) + self.assertLen(children, 1) + + child = children[0] + self.assertEqual( + child.flow_class_name, file_finder.ClientFileFinder.__name__ + ) + # We expect 7 condition-attributes to be converted + # to 7 FileFinderConditions. + self.assertLen(child.args.conditions, 7) + + def _GetCondition(condition_type): + for c in child.args.conditions: + if c.condition_type == condition_type: + return c.UnionCast() + + raise RuntimeError(f"Condition of type {condition_type} not found.") + + self.assertEqual( + _GetCondition( + rdf_file_finder.FileFinderCondition.Type.MODIFICATION_TIME + ), + modification_time, + ) + + self.assertEqual( + _GetCondition(rdf_file_finder.FileFinderCondition.Type.ACCESS_TIME), + access_time, + ) + + self.assertEqual( + _GetCondition( + rdf_file_finder.FileFinderCondition.Type.INODE_CHANGE_TIME + ), + inode_change_time, + ) + + self.assertEqual( + _GetCondition(rdf_file_finder.FileFinderCondition.Type.SIZE), size + ) + + self.assertEqual( + _GetCondition(rdf_file_finder.FileFinderCondition.Type.EXT_FLAGS), + ext_flags, + ) + + self.assertEqual( + _GetCondition( + rdf_file_finder.FileFinderCondition.Type.CONTENTS_REGEX_MATCH + ), + contents_regex_match, + ) + + self.assertEqual( + _GetCondition( + rdf_file_finder.FileFinderCondition.Type.CONTENTS_LITERAL_MATCH + ), + contents_literal_match, + ) + + def main(argv): # Run the full test suite test_lib.main(argv) diff --git a/grr/server/grr_response_server/flows/general/ca_enroller.py b/grr/server/grr_response_server/flows/general/ca_enroller.py deleted file mode 100644 index fcf7a59110..0000000000 --- a/grr/server/grr_response_server/flows/general/ca_enroller.py +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env python -"""A flow to enrol new clients.""" - -import logging -from grr_response_core.lib import rdfvalue -from grr_response_core.lib import utils -from grr_response_core.lib.rdfvalues import client as rdf_client -from grr_response_core.lib.rdfvalues import crypto as rdf_crypto -from grr_response_core.lib.rdfvalues import structs as rdf_structs -from grr_response_proto import flows_pb2 -from grr_response_server import client_index -from grr_response_server import data_store -from grr_response_server import events -from grr_response_server import flow -from grr_response_server import flow_base -from grr_response_server import message_handlers -from grr_response_server.rdfvalues import objects as rdf_objects - - -class CAEnrolerArgs(rdf_structs.RDFProtoStruct): - protobuf = flows_pb2.CAEnrolerArgs - rdf_deps = [ - rdf_crypto.Certificate, - ] - - -class CAEnroler(flow_base.FlowBase): - """Enrol new clients.""" - - args_type = CAEnrolerArgs - - def Start(self): - """Sign the CSR from the client.""" - if self.args.csr.type != rdf_crypto.Certificate.Type.CSR: - raise ValueError("Must be called with CSR") - - csr = rdf_crypto.CertificateSigningRequest(self.args.csr.pem) - # Verify the CSR. This is not strictly necessary but doesn't harm either. - try: - csr.Verify(csr.GetPublicKey()) - except rdf_crypto.VerificationError: - raise flow_base.FlowError("CSR for client %s did not verify: %s" % - (self.client_id, csr.AsPEM())) - - # Verify that the CN is of the correct form. The common name should refer - # to a client URN. - self.cn = rdf_client.ClientURN.FromPublicKey(csr.GetPublicKey()) - if self.cn != csr.GetCN(): - raise ValueError("CSR CN %s does not match public key %s." % - (csr.GetCN(), self.cn)) - - logging.info("Will sign CSR for: %s", self.cn) - - cert = rdf_crypto.RDFX509Cert.ClientCertFromCSR(csr) - - # This check is important to ensure that the client id reported in the - # source of the enrollment request is the same as the one in the - # certificate. We use the ClientURN to ensure this is also of the correct - # form for a client name. - if self.cn != self.client_id: - raise flow_base.FlowError("Certificate name %s mismatch for client %s" % - (self.cn, self.client_id)) - - data_store.REL_DB.WriteClientMetadata( - self.client_id, certificate=cert, fleetspeak_enabled=False) - index = client_index.ClientIndex() - index.AddClient(rdf_objects.ClientSnapshot(client_id=self.client_id)) - - # Publish the client enrollment message. - events.Events.PublishEvent( - "ClientEnrollment", self.client_urn, username=self.creator) - - self.Log("Enrolled %s successfully", self.client_id) - - -enrolment_cache = utils.FastStore(5000) - - -class EnrolmentHandler(message_handlers.MessageHandler): - """Message handler to process enrolment requests.""" - - handler_name = "Enrol" - - def ProcessMessages(self, msgs): - client_ids = set() - requests = {} - - for msg in msgs: - client_id = msg.client_id - - # It makes no sense to enrol the same client multiple times, so we - # eliminate duplicates. Note, that we can still enroll clients multiple - # times due to cache expiration. - try: - enrolment_cache.Get(client_id) - continue - except KeyError: - enrolment_cache.Put(client_id, 1) - client_ids.add(client_id) - requests[client_id] = msg.request.payload - - if not client_ids: - return - - try: - mds = data_store.REL_DB.MultiReadClientMetadata(list(client_ids)) - for client_id in client_ids: - if client_id not in mds or not mds[client_id].certificate: - # Start the enrollment flow for this client. - - # As mentioned in the comment above, the CAEnroler class is - # autogenerated which confuses the linter. - data_store.REL_DB.WriteClientMetadata( - client_id, - first_seen=rdfvalue.RDFDatetime.Now(), - fleetspeak_enabled=False) - flow.StartFlow( - client_id=client_id, - flow_cls=CAEnroler, - creator="GRRWorker", - csr=requests[client_id]) - except Exception as e: # pylint: disable=broad-except - logging.exception("Exception while starting interrogate: %s", e) diff --git a/grr/server/grr_response_server/flows/general/collectors.py b/grr/server/grr_response_server/flows/general/collectors.py index b3bed623be..fe49841011 100644 --- a/grr/server/grr_response_server/flows/general/collectors.py +++ b/grr/server/grr_response_server/flows/general/collectors.py @@ -38,12 +38,14 @@ def _ReadClientKnowledgeBase(client_id, allow_uninitialized=False): client = data_store.REL_DB.ReadClientSnapshot(client_id) return artifact.GetKnowledgeBase( - client, allow_uninitialized=allow_uninitialized) + client, allow_uninitialized=allow_uninitialized + ) -def _GetPathType(args: rdf_artifacts.ArtifactCollectorFlowArgs, - client_os: str) -> rdf_paths.PathSpec.PathType: - if args.use_tsk or args.use_raw_filesystem_access: +def _GetPathType( + args: rdf_artifacts.ArtifactCollectorFlowArgs, client_os: str +) -> rdf_paths.PathSpec.PathType: + if args.use_raw_filesystem_access: if client_os == "Windows": return config.CONFIG["Server.raw_filesystem_access_pathtype"] else: @@ -53,7 +55,7 @@ def _GetPathType(args: rdf_artifacts.ArtifactCollectorFlowArgs, def _GetImplementationType( - args: rdf_artifacts.ArtifactCollectorFlowArgs + args: rdf_artifacts.ArtifactCollectorFlowArgs, ) -> rdf_paths.PathSpec.ImplementationType: if args.HasField("implementation_type"): return args.implementation_type @@ -111,19 +113,18 @@ def Start(self): self.state.response_count = 0 self.state.progress = rdf_artifacts.ArtifactCollectorFlowProgress() - if self.args.use_tsk and self.args.use_raw_filesystem_access: - raise ValueError( - "Only one of use_tsk and use_raw_filesystem_access can be set.") - - if (self.args.dependencies == - rdf_artifacts.ArtifactCollectorFlowArgs.Dependency.FETCH_NOW): + if ( + self.args.dependencies + == rdf_artifacts.ArtifactCollectorFlowArgs.Dependency.FETCH_NOW + ): # String due to dependency loop with discover.py. self.CallFlow("Interrogate", next_state=self.StartCollection.__name__) return - elif (self.args.dependencies - == rdf_artifacts.ArtifactCollectorFlowArgs.Dependency.USE_CACHED - ) and (not self.state.knowledge_base): + elif ( + self.args.dependencies + == rdf_artifacts.ArtifactCollectorFlowArgs.Dependency.USE_CACHED + ) and (not self.state.knowledge_base): # If not provided, get a knowledge base from the client. try: self.state.knowledge_base = _ReadClientKnowledgeBase(self.client_id) @@ -152,11 +153,13 @@ def StartCollection(self, responses): """Start collecting.""" if not responses.success: raise artifact_utils.KnowledgeBaseUninitializedError( - "Attempt to initialize Knowledge Base failed.") + "Attempt to initialize Knowledge Base failed." + ) if not self.state.knowledge_base: self.state.knowledge_base = _ReadClientKnowledgeBase( - self.client_id, allow_uninitialized=True) + self.client_id, allow_uninitialized=True + ) for artifact_name in self.args.artifact_list: artifact_obj = self._GetArtifactFromName(artifact_name) @@ -175,87 +178,83 @@ def Collect(self, artifact_obj): # Ensure attempted artifacts are shown in progress, even with 0 results. self._GetOrInsertArtifactProgress(artifact_name) - test_conditions = list(artifact_obj.conditions) - os_conditions = ConvertSupportedOSToConditions(artifact_obj) - if os_conditions: - test_conditions.append(os_conditions) - - # Check each of the conditions match our target. - for condition in test_conditions: - if not artifact_utils.CheckCondition(condition, - self.state.knowledge_base): - logging.debug("Artifact %s condition %s failed on %s", artifact_name, - condition, self.client_id) - self.state.artifacts_skipped_due_to_condition.append( - (artifact_name, condition)) - return + if not MeetsOSConditions(self.state.knowledge_base, artifact_obj): + logging.debug( + "%s: Artifact %s not supported on os %s (options %s)", + self.client_id, + artifact_name, + self.state.knowledge_base.os, + artifact_obj.supported_os, + ) + self.state.artifacts_skipped_due_to_condition.append( + (artifact_name, "OS") + ) + return + sources_ran = 0 # Call the source defined action for each source. for source in artifact_obj.sources: - # Check conditions on the source. - source_conditions_met = True - test_conditions = list(source.conditions) - os_conditions = ConvertSupportedOSToConditions(source) - if os_conditions: - test_conditions.append(os_conditions) - - for condition in test_conditions: - if not artifact_utils.CheckCondition(condition, - self.state.knowledge_base): - source_conditions_met = False - - if source_conditions_met: - type_name = source.type - source_type = rdf_artifacts.ArtifactSource.SourceType - self.current_artifact_name = artifact_name - if type_name == source_type.COMMAND: - self.RunCommand(source) - # TODO(hanuszczak): `DIRECTORY` is deprecated [1], it should be removed. - # - # [1]: https://github.com/ForensicArtifacts/artifacts/pull/475 - elif (type_name == source_type.DIRECTORY or - type_name == source_type.PATH): - self.GetPaths( - source, - _GetPathType(self.args, self.client_os), - _GetImplementationType(self.args), - rdf_file_finder.FileFinderAction.Stat(), - ) - elif type_name == source_type.FILE: - self.GetPaths( - source, - _GetPathType(self.args, self.client_os), - _GetImplementationType(self.args), - rdf_file_finder.FileFinderAction.Download( - max_size=self.args.max_file_size - ), - ) - elif type_name == source_type.GREP: - self.Grep(source, _GetPathType(self.args, self.client_os), - _GetImplementationType(self.args)) - elif type_name == source_type.REGISTRY_KEY: - self.GetRegistryKey(source) - elif type_name == source_type.REGISTRY_VALUE: - self.GetRegistryValue(source) - elif type_name == source_type.WMI: - self.WMIQuery(source) - elif type_name == source_type.REKALL_PLUGIN: - raise NotImplementedError( - "Running Rekall artifacts is not supported anymore.") - elif type_name == source_type.ARTIFACT_GROUP: - self.CollectArtifacts(source) - elif type_name == source_type.ARTIFACT_FILES: - self.CollectArtifactFiles(source) - elif type_name == source_type.GRR_CLIENT_ACTION: - self.RunGrrClientAction(source) - else: - raise RuntimeError("Invalid type %s in %s" % - (type_name, artifact_name)) + if not MeetsOSConditions(self.state.knowledge_base, source): + continue + sources_ran += 1 + + type_name = source.type + source_type = rdf_artifacts.ArtifactSource.SourceType + self.current_artifact_name = artifact_name + if type_name == source_type.COMMAND: + self.RunCommand(source) + # TODO(hanuszczak): `DIRECTORY` is deprecated [1], it should be removed. + # + # [1]: https://github.com/ForensicArtifacts/artifacts/pull/475 + elif type_name == source_type.DIRECTORY or type_name == source_type.PATH: + self.GetPaths( + source, + _GetPathType(self.args, self.client_os), + _GetImplementationType(self.args), + rdf_file_finder.FileFinderAction.Stat(), + ) + elif type_name == source_type.FILE: + self.GetPaths( + source, + _GetPathType(self.args, self.client_os), + _GetImplementationType(self.args), + rdf_file_finder.FileFinderAction.Download( + max_size=self.args.max_file_size + ), + ) + elif type_name == source_type.GREP: + self.Grep( + source, + _GetPathType(self.args, self.client_os), + _GetImplementationType(self.args), + ) + elif type_name == source_type.REGISTRY_KEY: + self.GetRegistryKey(source) + elif type_name == source_type.REGISTRY_VALUE: + self.GetRegistryValue(source) + elif type_name == source_type.WMI: + self.WMIQuery(source) + elif type_name == source_type.REKALL_PLUGIN: + raise NotImplementedError( + "Running Rekall artifacts is not supported anymore." + ) + elif type_name == source_type.ARTIFACT_GROUP: + self.CollectArtifacts(source) + elif type_name == source_type.ARTIFACT_FILES: + self.CollectArtifactFiles(source) + elif type_name == source_type.GRR_CLIENT_ACTION: + self.RunGrrClientAction(source) else: - logging.debug( - "Artifact %s no sources run due to all sources " - "having failing conditions on %s", artifact_name, self.client_id) + raise RuntimeError("Invalid type %s in %s" % (type_name, artifact_name)) + + if sources_ran == 0: + logging.debug( + "Artifact %s no sources run due to all sources " + "having failing conditions on %s", + artifact_name, + self.client_id, + ) def _AreArtifactsKnowledgeBaseArtifacts(self): knowledgebase_list = config.CONFIG["Artifacts.knowledge_base"] @@ -284,13 +283,15 @@ def GetPaths(self, source, path_type, implementation_type, action): def ProcessFileFinderResults(self, responses): if not responses.success: - self.Log("Failed to fetch files %s" % - responses.request_data["artifact_name"]) + self.Log( + "Failed to fetch files %s" % responses.request_data["artifact_name"] + ) else: self.CallStateInline( next_state=self.ProcessCollected.__name__, request_data=responses.request_data, - messages=[r.stat_entry for r in responses]) + messages=[r.stat_entry for r in responses], + ) def Glob(self, source, pathtype, implementation_type): """Glob paths, return StatEntry objects.""" @@ -301,9 +302,10 @@ def Glob(self, source, pathtype, implementation_type): implementation_type=implementation_type, request_data={ "artifact_name": self.current_artifact_name, - "source": source.ToPrimitiveDict() + "source": source.ToPrimitiveDict(), }, - next_state=self.ProcessCollected.__name__) + next_state=self.ProcessCollected.__name__, + ) def _CombineRegex(self, regex_list): if len(regex_list) == 1: @@ -347,12 +349,15 @@ def Grep(self, source, pathtype, implementation_type): regex=self._CombineRegex(content_regex_list), bytes_before=0, bytes_after=0, - mode="ALL_HITS") + mode="ALL_HITS", + ) file_finder_condition = rdf_file_finder.FileFinderCondition( condition_type=( - rdf_file_finder.FileFinderCondition.Type.CONTENTS_REGEX_MATCH), - contents_regex_match=regex_condition) + rdf_file_finder.FileFinderCondition.Type.CONTENTS_REGEX_MATCH + ), + contents_regex_match=regex_condition, + ) self.CallFlow( file_finder.FileFinder.__name__, @@ -363,9 +368,10 @@ def Grep(self, source, pathtype, implementation_type): implementation_type=implementation_type, request_data={ "artifact_name": self.current_artifact_name, - "source": source.ToPrimitiveDict() + "source": source.ToPrimitiveDict(), }, - next_state=self.ProcessCollected.__name__) + next_state=self.ProcessCollected.__name__, + ) def GetRegistryKey(self, source): self.CallFlow( @@ -374,9 +380,10 @@ def GetRegistryKey(self, source): pathtype=rdf_paths.PathSpec.PathType.REGISTRY, request_data={ "artifact_name": self.current_artifact_name, - "source": source.ToPrimitiveDict() + "source": source.ToPrimitiveDict(), }, - next_state=self.ProcessCollected.__name__) + next_state=self.ProcessCollected.__name__, + ) def GetRegistryValue(self, source): """Retrieve directly specified registry values, returning Stat objects.""" @@ -384,7 +391,8 @@ def GetRegistryValue(self, source): has_glob = False for kvdict in source.attributes["key_value_pairs"]: if "*" in kvdict["key"] or rdf_paths.GROUPING_PATTERN.search( - kvdict["key"]): + kvdict["key"] + ): has_glob = True if kvdict["value"]: @@ -399,7 +407,8 @@ def GetRegistryValue(self, source): try: expanded_paths = artifact_utils.InterpolateKbAttributes( - path, self.state.knowledge_base) + path, self.state.knowledge_base + ) except artifact_utils.KbInterpolationMissingAttributesError as error: logging.error(str(error)) if not self.args.ignore_interpolation_errors: @@ -415,16 +424,18 @@ def GetRegistryValue(self, source): pathtype=rdf_paths.PathSpec.PathType.REGISTRY, request_data={ "artifact_name": self.current_artifact_name, - "source": source.ToPrimitiveDict() + "source": source.ToPrimitiveDict(), }, - next_state=self.ProcessCollected.__name__) + next_state=self.ProcessCollected.__name__, + ) else: # We call statfile directly for keys that don't include globs because it # is faster and some artifacts rely on getting an IOError to trigger # fallback processing. for new_path in new_paths: pathspec = rdf_paths.PathSpec( - path=new_path, pathtype=rdf_paths.PathSpec.PathType.REGISTRY) + path=new_path, pathtype=rdf_paths.PathSpec.PathType.REGISTRY + ) self.CallClient( server_stubs.GetFileStat, @@ -440,8 +451,7 @@ def _StartSubArtifactCollector(self, artifact_list, source, next_state): self.CallFlow( ArtifactCollectorFlow.__name__, artifact_list=artifact_list, - use_raw_filesystem_access=(self.args.use_raw_filesystem_access or - self.args.use_tsk), + use_raw_filesystem_access=self.args.use_raw_filesystem_access, apply_parsers=self.args.apply_parsers, max_file_size=self.args.max_file_size, ignore_interpolation_errors=self.args.ignore_interpolation_errors, @@ -449,22 +459,25 @@ def _StartSubArtifactCollector(self, artifact_list, source, next_state): knowledge_base=self.args.knowledge_base, request_data={ "artifact_name": self.current_artifact_name, - "source": source.ToPrimitiveDict() + "source": source.ToPrimitiveDict(), }, - next_state=next_state) + next_state=next_state, + ) def CollectArtifacts(self, source): self._StartSubArtifactCollector( artifact_list=source.attributes["names"], source=source, - next_state=self.ProcessCollected.__name__) + next_state=self.ProcessCollected.__name__, + ) def CollectArtifactFiles(self, source): """Collect files from artifact pathspecs.""" self._StartSubArtifactCollector( artifact_list=source.attributes["artifact_list"], source=source, - next_state=self.ProcessCollectedArtifactFiles.__name__) + next_state=self.ProcessCollectedArtifactFiles.__name__, + ) def RunCommand(self, source): """Run a command.""" @@ -474,9 +487,10 @@ def RunCommand(self, source): args=source.attributes.get("args", []), request_data={ "artifact_name": self.current_artifact_name, - "source": source.ToPrimitiveDict() + "source": source.ToPrimitiveDict(), }, - next_state=self.ProcessCollected.__name__) + next_state=self.ProcessCollected.__name__, + ) def WMIQuery(self, source): """Run a Windows WMI Query.""" @@ -490,16 +504,18 @@ def WMIQuery(self, source): base_object=base_object, request_data={ "artifact_name": self.current_artifact_name, - "source": source.ToPrimitiveDict() + "source": source.ToPrimitiveDict(), }, - next_state=self.ProcessCollected.__name__) + next_state=self.ProcessCollected.__name__, + ) def _GetSingleExpansion(self, value): results = list(self._Interpolate(value)) if len(results) > 1: - raise ValueError("Interpolation generated multiple results, use a" - " list for multi-value expansions. %s yielded: %s" % - (value, results)) + raise ValueError( + "Interpolation generated multiple results, use a" + " list for multi-value expansions. %s yielded: %s" % (value, results) + ) return results[0] def InterpolateDict(self, input_dict): @@ -549,10 +565,11 @@ def InterpolateList(self, input_list): new_args.extend(value) return new_args - def _Interpolate(self, - pattern: Text, - knowledgebase: Optional[rdf_client.KnowledgeBase] = None)\ - -> Sequence[Text]: + def _Interpolate( + self, + pattern: Text, + knowledgebase: Optional[rdf_client.KnowledgeBase] = None, + ) -> Sequence[Text]: """Performs a knowledgebase interpolation. Args: @@ -590,31 +607,38 @@ def RunGrrClientAction(self, source): action_stub, request_data={ "artifact_name": self.current_artifact_name, - "source": source.ToPrimitiveDict() + "source": source.ToPrimitiveDict(), }, next_state=self.ProcessCollected.__name__, - **self.InterpolateDict(source.attributes.get("action_args", {}))) + **self.InterpolateDict(source.attributes.get("action_args", {})), + ) def CallFallback(self, artifact_name, request_data): - if artifact_name not in artifact_fallbacks.FALLBACK_REGISTRY: return False fallback_flow = artifact_fallbacks.FALLBACK_REGISTRY[artifact_name] if artifact_name in self.state.called_fallbacks: - self.Log("Already called fallback class %s for artifact: %s", - fallback_flow, artifact_name) + self.Log( + "Already called fallback class %s for artifact: %s", + fallback_flow, + artifact_name, + ) return False - self.Log("Calling fallback class %s for artifact: %s", fallback_flow, - artifact_name) + self.Log( + "Calling fallback class %s for artifact: %s", + fallback_flow, + artifact_name, + ) self.CallFlow( fallback_flow, request_data=request_data.ToDict(), artifact_name=artifact_name, - next_state=self.ProcessCollected.__name__) + next_state=self.ProcessCollected.__name__, + ) # Make sure we only try this once self.state.called_fallbacks.add(artifact_name) @@ -637,10 +661,17 @@ def ProcessCollected(self, responses): if responses.success: self.Log( "Artifact data collection %s completed successfully in flow %s " - "with %d responses", artifact_name, flow_name, len(responses)) + "with %d responses", + artifact_name, + flow_name, + len(responses), + ) else: - self.Log("Artifact %s data collection failed. Status: %s.", artifact_name, - responses.status) + self.Log( + "Artifact %s data collection failed. Status: %s.", + artifact_name, + responses.status, + ) # If the ArtifactDescriptor specifies a fallback for the failed Artifact, # call the fallback without processing any responses of the failed @@ -666,7 +697,8 @@ def ProcessCollectedRegistryStatEntry(self, responses): """ if not responses.success: self.CallStateInline( - next_state=self.ProcessCollected.__name__, responses=responses) + next_state=self.ProcessCollected.__name__, responses=responses + ) return stat_entries = list(map(rdf_client_fs.StatEntry, responses)) @@ -675,7 +707,8 @@ def ProcessCollectedRegistryStatEntry(self, responses): self.CallStateInline( next_state=self.ProcessCollected.__name__, request_data=responses.request_data, - messages=stat_entries) + messages=stat_entries, + ) def ProcessCollectedArtifactFiles(self, responses): """Schedule files for download based on pathspec attribute. @@ -696,8 +729,9 @@ def ProcessCollectedArtifactFiles(self, responses): if response.HasField(pathspec_attribute): pathspec = response.Get(pathspec_attribute) else: - self.Log("Missing pathspec field %s: %s", pathspec_attribute, - response) + self.Log( + "Missing pathspec field %s: %s", pathspec_attribute, response + ) continue else: pathspec = response @@ -716,7 +750,7 @@ def ProcessCollectedArtifactFiles(self, responses): if not pathspec.path: self.Log("Skipping empty pathspec.") continue - if self.args.use_raw_filesystem_access or self.args.use_tsk: + if self.args.use_raw_filesystem_access: pathspec.pathtype = rdf_paths.PathSpec.PathType.TSK else: pathspec.pathtype = rdf_paths.PathSpec.PathType.OS @@ -726,7 +760,8 @@ def ProcessCollectedArtifactFiles(self, responses): else: raise RuntimeError( "Response must be a string path, a pathspec, or have " - "pathspec_attribute set. Got: %s" % pathspec) + "pathspec_attribute set. Got: %s" % pathspec + ) if self.download_list: request_data = responses.request_data.ToDict() @@ -734,7 +769,8 @@ def ProcessCollectedArtifactFiles(self, responses): transfer.MultiGetFile.__name__, pathspecs=self.download_list, request_data=request_data, - next_state=self.ProcessCollected.__name__) + next_state=self.ProcessCollected.__name__, + ) else: self.Log("No files to download") @@ -755,8 +791,9 @@ def _ParseResponses(self, responses, artifact_name, source): if self.args.apply_parsers: parser_factory = parsers.ArtifactParserFactory(artifact_name) - results = artifact.ApplyParsersToResponses(parser_factory, responses, - self) + results = artifact.ApplyParsersToResponses( + parser_factory, responses, self + ) else: results = responses @@ -768,15 +805,16 @@ def _ParseResponses(self, responses, artifact_name, source): result_type = result.__class__.__name__ if result_type == "Anomaly": self.SendReply(result) - elif (not artifact_return_types or result_type in artifact_return_types): + elif not artifact_return_types or result_type in artifact_return_types: self.state.response_count += 1 self.SendReply(result, tag="artifact:%s" % artifact_name) def GetProgress(self) -> rdf_artifacts.ArtifactCollectorFlowProgress: return self.state.progress - def _GetOrInsertArtifactProgress(self, - name: str) -> rdf_artifacts.ArtifactProgress: + def _GetOrInsertArtifactProgress( + self, name: str + ) -> rdf_artifacts.ArtifactProgress: try: return next(a for a in self.state.progress.artifacts if a.name == name) except StopIteration: @@ -789,7 +827,8 @@ def End(self, responses): # If we got no responses, and user asked for it, we error out. if self.args.error_on_no_results and self.state.response_count == 0: raise artifact_utils.ArtifactProcessingError( - "Artifact collector returned 0 responses.") + "Artifact collector returned 0 responses." + ) class ArtifactFilesDownloaderFlowArgs(rdf_structs.RDFProtoStruct): @@ -812,8 +851,9 @@ def GetOriginalResultType(self): return rdfvalue.RDFValue.classes.get(self.original_result_type) -class ArtifactFilesDownloaderFlow(transfer.MultiGetFileLogic, - flow_base.FlowBase): +class ArtifactFilesDownloaderFlow( + transfer.MultiGetFileLogic, flow_base.FlowBase +): """Flow that downloads files referenced by collected artifacts.""" category = "/Collectors/" @@ -824,17 +864,18 @@ def _FindMatchingPathspecs(self, response): # If we're dealing with plain file StatEntry, just # return it's pathspec - there's nothing to parse # and guess. - if (isinstance(response, rdf_client_fs.StatEntry) and - response.pathspec.pathtype in [ - rdf_paths.PathSpec.PathType.TSK, - rdf_paths.PathSpec.PathType.OS, - rdf_paths.PathSpec.PathType.NTFS, - ]): + if isinstance( + response, rdf_client_fs.StatEntry + ) and response.pathspec.pathtype in [ + rdf_paths.PathSpec.PathType.TSK, + rdf_paths.PathSpec.PathType.OS, + rdf_paths.PathSpec.PathType.NTFS, + ]: return [response.pathspec] knowledge_base = _ReadClientKnowledgeBase(self.client_id) - if self.args.use_raw_filesystem_access or self.args.use_tsk: + if self.args.use_raw_filesystem_access: path_type = rdf_paths.PathSpec.PathType.TSK else: path_type = rdf_paths.PathSpec.PathType.OS @@ -851,10 +892,6 @@ def _FindMatchingPathspecs(self, response): def Start(self): super().Start() - if self.args.use_tsk and self.args.use_raw_filesystem_access: - raise ValueError( - "Only one of use_tsk and use_raw_filesystem_access can be set.") - self.state.file_size = self.args.max_file_size self.state.results_to_download = [] @@ -867,10 +904,10 @@ def Start(self): ArtifactCollectorFlow.__name__, next_state=self._DownloadFiles.__name__, artifact_list=self.args.artifact_list, - use_raw_filesystem_access=(self.args.use_tsk or - self.args.use_raw_filesystem_access), + use_raw_filesystem_access=self.args.use_raw_filesystem_access, implementation_type=implementation_type, - max_file_size=self.args.max_file_size) + max_file_size=self.args.max_file_size, + ) def _DownloadFiles(self, responses): if not responses.success: @@ -886,28 +923,30 @@ def _DownloadFiles(self, responses): result = ArtifactFilesDownloaderResult( original_result_type=response.__class__.__name__, original_result=response, - found_pathspec=pathspec) + found_pathspec=pathspec, + ) results_with_pathspecs.append(result) else: result = ArtifactFilesDownloaderResult( original_result_type=response.__class__.__name__, - original_result=response) + original_result=response, + ) results_without_pathspecs.append(result) grouped_results = collection.Group( - results_with_pathspecs, lambda x: x.found_pathspec.CollapsePath()) + results_with_pathspecs, lambda x: x.found_pathspec.CollapsePath() + ) for _, group in grouped_results.items(): self.StartFileFetch( - group[0].found_pathspec, request_data=dict(results=group)) + group[0].found_pathspec, request_data=dict(results=group) + ) for result in results_without_pathspecs: self.SendReply(result) - def ReceiveFetchedFile(self, - stat_entry, - file_hash, - request_data=None, - is_duplicate=False): + def ReceiveFetchedFile( + self, stat_entry, file_hash, request_data=None, is_duplicate=False + ): """See MultiGetFileLogic.""" del is_duplicate # Unused. @@ -948,8 +987,10 @@ def Start(self): self.CallFlow("Interrogate", next_state=self.StartCollection.__name__) return - if (self.args.dependencies == dependency.USE_CACHED and - not self.state.knowledge_base): + if ( + self.args.dependencies == dependency.USE_CACHED + and not self.state.knowledge_base + ): # If not provided, get a knowledge base from the client. try: self.state.knowledge_base = _ReadClientKnowledgeBase(self.client_id) @@ -959,7 +1000,8 @@ def Start(self): if not self._AreArtifactsKnowledgeBaseArtifacts(): # String due to dependency loop with discover.py self.CallFlow( - "Interrogate", next_state=self.StartCollection.__name__) + "Interrogate", next_state=self.StartCollection.__name__ + ) return # In all other cases start the collection state. @@ -969,13 +1011,15 @@ def StartCollection(self, responses): """Start collecting.""" if not responses.success: raise artifact_utils.KnowledgeBaseUninitializedError( - "Attempt to initialize Knowledge Base failed.") + "Attempt to initialize Knowledge Base failed." + ) # TODO(hanuszczak): Flow arguments also appear to have some knowledgebase. # Do we use it in any way? if not self.state.knowledge_base: self.state.knowledge_base = _ReadClientKnowledgeBase( - self.client_id, allow_uninitialized=True) + self.client_id, allow_uninitialized=True + ) request = GetArtifactCollectorArgs(self.args, self.state.knowledge_base) self.CollectArtifacts(request) @@ -992,14 +1036,18 @@ def CollectArtifacts(self, client_artifact_collector_args): self.CallClient( server_stubs.ArtifactCollector, request=client_artifact_collector_args, - next_state=self.ProcessCollected.__name__) + next_state=self.ProcessCollected.__name__, + ) def ProcessCollected(self, responses): flow_name = self.__class__.__name__ if responses.success: self.Log( "Artifact data collection completed successfully in flow %s " - "with %d responses", flow_name, len(responses)) + "with %d responses", + flow_name, + len(responses), + ) else: self.Log("Artifact data collection failed. Status: %s.", responses.status) return @@ -1021,14 +1069,8 @@ def End(self, responses): # If we got no responses, and user asked for it, we error out. if self.args.error_on_no_results and self.state.response_count == 0: raise artifact_utils.ArtifactProcessingError( - "Artifact collector returned 0 responses.") - - -def ConvertSupportedOSToConditions(src_object): - """Turn supported_os into a condition.""" - if src_object.supported_os: - conditions = " OR ".join("os == '%s'" % o for o in src_object.supported_os) - return conditions + "Artifact collector returned 0 responses." + ) def GetArtifactCollectorArgs(flow_args, knowledge_base): @@ -1052,15 +1094,18 @@ def GetArtifactCollectorArgs(flow_args, knowledge_base): if not flow_args.recollect_knowledge_base: artifact_names = flow_args.artifact_list else: - artifact_names = GetArtifactsForCollection(knowledge_base.os, - flow_args.artifact_list) + artifact_names = GetArtifactsForCollection( + knowledge_base.os, flow_args.artifact_list + ) - expander = ArtifactExpander(knowledge_base, - _GetPathType(flow_args, knowledge_base.os), - flow_args.max_file_size) + expander = ArtifactExpander( + knowledge_base, + _GetPathType(flow_args, knowledge_base.os), + flow_args.max_file_size, + ) for artifact_name in artifact_names: rdf_artifact = artifact_registry.REGISTRY.GetArtifact(artifact_name) - if not MeetsConditions(knowledge_base, rdf_artifact): + if not MeetsOSConditions(knowledge_base, rdf_artifact): continue if artifact_name in expander.processed_artifacts: continue @@ -1070,17 +1115,12 @@ def GetArtifactCollectorArgs(flow_args, knowledge_base): return args -def MeetsConditions(knowledge_base, source): - """Check conditions on the source.""" - source_conditions_met = True - os_conditions = ConvertSupportedOSToConditions(source) - if os_conditions: - source.conditions.append(os_conditions) - for condition in source.conditions: - source_conditions_met &= artifact_utils.CheckCondition( - condition, knowledge_base) +def MeetsOSConditions(knowledge_base, source): + """Check supported OS on the source.""" + if source.supported_os and knowledge_base.os not in source.supported_os: + return False - return source_conditions_met + return True class ArtifactExpander(object): @@ -1122,24 +1162,28 @@ def Expand(self, rdf_artifact, requested): expanded_artifact = rdf_artifacts.ExpandedArtifact( name=rdf_artifact.name, provides=rdf_artifact.provides, - requested_by_user=requested) + requested_by_user=requested, + ) for source in rdf_artifact.sources: - if MeetsConditions(self._knowledge_base, source): - type_name = source.type + if not MeetsOSConditions(self._knowledge_base, source): + continue - if type_name == source_type.ARTIFACT_GROUP: - for subartifact in self._ExpandArtifactGroupSource(source, requested): - yield subartifact - continue + type_name = source.type - elif type_name == source_type.ARTIFACT_FILES: - expanded_sources = self._ExpandArtifactFilesSource(source, requested) + if type_name == source_type.ARTIFACT_GROUP: + for subartifact in self._ExpandArtifactGroupSource(source, requested): + yield subartifact + continue - else: - expanded_sources = self._ExpandBasicSource(source) + elif type_name == source_type.ARTIFACT_FILES: + expanded_sources = self._ExpandArtifactFilesSource(source, requested) + + else: + expanded_sources = self._ExpandBasicSource(source) + + expanded_artifact.sources.Extend(expanded_sources) - expanded_artifact.sources.Extend(expanded_sources) self.processed_artifacts.add(rdf_artifact.name) if expanded_artifact.sources: yield expanded_artifact @@ -1148,7 +1192,8 @@ def _ExpandBasicSource(self, source): expanded_source = rdf_artifacts.ExpandedSource( base_source=source, path_type=self._path_type, - max_bytesize=self._max_file_size) + max_bytesize=self._max_file_size, + ) return [expanded_source] def _ExpandArtifactGroupSource(self, source, requested): @@ -1224,7 +1269,8 @@ def _InitializeGraph(self, os_name, artifact_list): artifact_list: List of requested artifact names. """ dependencies = artifact_registry.REGISTRY.SearchDependencies( - os_name, artifact_list) + os_name, artifact_list + ) artifact_names, attribute_names = dependencies self._AddAttributeNodes(attribute_names) @@ -1279,7 +1325,8 @@ def _AddDependencyEdges(self, rdf_artifact): rdf_artifact: The artifact object. """ artifact_dependencies = artifact_registry.GetArtifactPathDependencies( - rdf_artifact) + rdf_artifact + ) if artifact_dependencies: for attribute in artifact_dependencies: self._AddEdge(attribute, rdf_artifact.name) diff --git a/grr/server/grr_response_server/flows/general/collectors_test.py b/grr/server/grr_response_server/flows/general/collectors_test.py index a329c413d2..2ada6c38df 100644 --- a/grr/server/grr_response_server/flows/general/collectors_test.py +++ b/grr/server/grr_response_server/flows/general/collectors_test.py @@ -61,8 +61,9 @@ def setUp(self): artifact_registry.REGISTRY.ClearSources() artifact_registry.REGISTRY.ClearRegistry() - test_artifacts_file = os.path.join(config.CONFIG["Test.data_dir"], - "artifacts", "test_artifacts.json") + test_artifacts_file = os.path.join( + config.CONFIG["Test.data_dir"], "artifacts", "test_artifacts.json" + ) artifact_registry.REGISTRY.AddFileSource(test_artifacts_file) self.fakeartifact = artifact_registry.REGISTRY.GetArtifact("FakeArtifact") @@ -71,14 +72,16 @@ def setUp(self): self.output_count = 0 -class TestArtifactCollectors(ArtifactCollectorsTestMixin, - flow_test_lib.FlowTestsBaseclass): +class TestArtifactCollectors( + ArtifactCollectorsTestMixin, flow_test_lib.FlowTestsBaseclass +): """Test the artifact collection mechanism with fake artifacts.""" def testInterpolateArgs(self): args = rdf_artifacts.ArtifactCollectorFlowArgs() collect_flow = collectors.ArtifactCollectorFlow( - rdf_flow_objects.Flow(args=args)) + rdf_flow_objects.Flow(args=args) + ) kb = rdf_client.KnowledgeBase() kb.MergeOrAddUser(rdf_client.User(username="test1")) @@ -91,24 +94,28 @@ def testInterpolateArgs(self): action_args = { "usernames": ["%%users.username%%", "%%users.username%%"], "nointerp": "asdfsdf", - "notastring": test_rdf + "notastring": test_rdf, } kwargs = collect_flow.InterpolateDict(action_args) - self.assertCountEqual(kwargs["usernames"], - ["test1", "test2", "test1", "test2"]) + self.assertCountEqual( + kwargs["usernames"], ["test1", "test2", "test1", "test2"] + ) self.assertEqual(kwargs["nointerp"], "asdfsdf") self.assertEqual(kwargs["notastring"], test_rdf) # We should be using an array since users.username will expand to multiple # values. - self.assertRaises(ValueError, collect_flow.InterpolateDict, - {"bad": "%%users.username%%"}) + self.assertRaises( + ValueError, collect_flow.InterpolateDict, {"bad": "%%users.username%%"} + ) list_args = collect_flow.InterpolateList( - ["%%users.username%%", r"%%users.username%%\aa"]) - self.assertCountEqual(list_args, - ["test1", "test2", r"test1\aa", r"test2\aa"]) + ["%%users.username%%", r"%%users.username%%\aa"] + ) + self.assertCountEqual( + list_args, ["test1", "test2", r"test1\aa", r"test2\aa"] + ) list_args = collect_flow.InterpolateList(["one"]) self.assertEqual(list_args, ["one"]) @@ -116,29 +123,33 @@ def testInterpolateArgs(self): # Ignore the failure in users.desktop, report the others. collect_flow.args.ignore_interpolation_errors = True list_args = collect_flow.InterpolateList( - ["%%users.desktop%%", r"%%users.username%%\aa"]) + ["%%users.desktop%%", r"%%users.username%%\aa"] + ) self.assertCountEqual(list_args, [r"test1\aa", r"test2\aa"]) # Both fail. list_args = collect_flow.InterpolateList( - [r"%%users.desktop%%\aa", r"%%users.sid%%\aa"]) + [r"%%users.desktop%%\aa", r"%%users.sid%%\aa"] + ) self.assertCountEqual(list_args, []) def testGrepRegexCombination(self): args = rdf_artifacts.ArtifactCollectorFlowArgs() collect_flow = collectors.ArtifactCollectorFlow( - rdf_flow_objects.Flow(args=args)) + rdf_flow_objects.Flow(args=args) + ) self.assertEqual(collect_flow._CombineRegex([b"simple"]), b"simple") self.assertEqual(collect_flow._CombineRegex([b"a", b"b"]), b"(a)|(b)") self.assertEqual( - collect_flow._CombineRegex([b"a", b"b", b"c"]), b"(a)|(b)|(c)") + collect_flow._CombineRegex([b"a", b"b", b"c"]), b"(a)|(b)|(c)" + ) self.assertEqual( collect_flow._CombineRegex([b"a|b", b"[^_]b", b"c|d"]), - b"(a|b)|([^_]b)|(c|d)") + b"(a|b)|([^_]b)|(c|d)", + ) def testGrep(self): - class MockCallFlow(object): def CallFlow(self, *args, **kwargs): @@ -146,14 +157,15 @@ def CallFlow(self, *args, **kwargs): self.kwargs = kwargs mock_call_flow = MockCallFlow() - with mock.patch.object(collectors.ArtifactCollectorFlow, "CallFlow", - mock_call_flow.CallFlow): - + with mock.patch.object( + collectors.ArtifactCollectorFlow, "CallFlow", mock_call_flow.CallFlow + ): args = mock.Mock() args.ignore_interpolation_errors = False collect_flow = collectors.ArtifactCollectorFlow( - rdf_flow_objects.Flow(args=args)) + rdf_flow_objects.Flow(args=args) + ) kb = rdf_client.KnowledgeBase() kb.MergeOrAddUser(rdf_client.User(username="test1")) kb.MergeOrAddUser(rdf_client.User(username="test2")) @@ -164,8 +176,9 @@ def CallFlow(self, *args, **kwargs): type=rdf_artifacts.ArtifactSource.SourceType.GREP, attributes={ "paths": ["/etc/passwd"], - "content_regex_list": [b"^a%%users.username%%b$"] - }) + "content_regex_list": [b"^a%%users.username%%b$"], + }, + ) collect_flow.Grep(collector, rdf_paths.PathSpec.PathType.TSK, None) conditions = mock_call_flow.kwargs["conditions"] @@ -180,7 +193,8 @@ def testGetArtifact(self): file_path = os.path.join(self.base_path, "win_hello.exe") coll1 = rdf_artifacts.ArtifactSource( type=rdf_artifacts.ArtifactSource.SourceType.FILE, - attributes={"paths": [file_path]}) + attributes={"paths": [file_path]}, + ) self.fakeartifact.sources.append(coll1) self._GetArtifact("FakeArtifact") @@ -204,7 +218,8 @@ def testArtifactUpload(self): """ % file_path artifact_obj = artifact_registry.REGISTRY.ArtifactsFromYaml( - artifact_source)[0] + artifact_source + )[0] artifact_registry.REGISTRY._CheckDirty() data_store.REL_DB.WriteArtifact(artifact_obj) @@ -228,7 +243,8 @@ def _GetArtifact(self, artifact_name): artifact_list=artifact_list, use_raw_filesystem_access=False, creator=self.test_username, - client_id=client_id) + client_id=client_id, + ) fd2 = open(file_path, "rb") fd2.seek(0, 2) @@ -239,32 +255,13 @@ def _GetArtifact(self, artifact_name): db.ClientPath( client_id, rdf_objects.PathInfo.PathType.OS, - components=tuple(components))) + components=tuple(components), + ) + ) fd.Seek(0, 2) size = fd.Tell() self.assertEqual(size, expected_size) - def testArtifactSkipping(self): - client_mock = action_mocks.ActionMock() - # This does not match the Artifact so it will not be collected. - client_id = self.SetupClient(0, system="Windows") - - artifact_list = ["FakeArtifact"] - flow_id = flow_test_lib.TestFlowHelper( - collectors.ArtifactCollectorFlow.__name__, - client_mock, - artifact_list=artifact_list, - use_raw_filesystem_access=False, - creator=self.test_username, - client_id=client_id) - - flow_obj = data_store.REL_DB.ReadFlowObject(client_id, flow_id) - state = flow_obj.persistent_data - - self.assertLen(state.artifacts_skipped_due_to_condition, 1) - self.assertEqual(state.artifacts_skipped_due_to_condition[0], - ["FakeArtifact", "os == 'Linux'"]) - def testRunGrrClientActionArtifact(self): """Test we can get a GRR client artifact.""" client_id = self.SetupClient(0, system="Linux") @@ -273,7 +270,8 @@ def testRunGrrClientActionArtifact(self): coll1 = rdf_artifacts.ArtifactSource( type=rdf_artifacts.ArtifactSource.SourceType.GRR_CLIENT_ACTION, - attributes={"client_action": standard.ListProcesses.__name__}) + attributes={"client_action": standard.ListProcesses.__name__}, + ) self.fakeartifact.sources.append(coll1) artifact_list = ["FakeArtifact"] flow_id = flow_test_lib.TestFlowHelper( @@ -281,61 +279,36 @@ def testRunGrrClientActionArtifact(self): client_mock, artifact_list=artifact_list, creator=self.test_username, - client_id=client_id) + client_id=client_id, + ) results = flow_test_lib.GetFlowResults(client_id, flow_id) self.assertIsInstance(results[0], rdf_client.Process) self.assertLen(results, 1) - def testConditions(self): - """Test we can get a GRR client artifact with conditions.""" - client_id = self.SetupClient(0, system="Linux") - with mock.patch.object(psutil, "process_iter", ProcessIter): - # Run with false condition. - client_mock = action_mocks.ActionMock(standard.ListProcesses) - coll1 = rdf_artifacts.ArtifactSource( - type=rdf_artifacts.ArtifactSource.SourceType.GRR_CLIENT_ACTION, - attributes={"client_action": standard.ListProcesses.__name__}, - conditions=["os == 'Windows'"]) - self.fakeartifact.sources.append(coll1) - results = self._RunClientActionArtifact(client_id, client_mock, - ["FakeArtifact"]) - self.assertEmpty(results) - - # Now run with matching or condition. - coll1.conditions = ["os == 'Linux' or os == 'Windows'"] - self.fakeartifact.sources = [] - self.fakeartifact.sources.append(coll1) - results = self._RunClientActionArtifact(client_id, client_mock, - ["FakeArtifact"]) - self.assertTrue(results) - - # Now run with impossible or condition. - coll1.conditions.append("os == 'NotTrue'") - self.fakeartifact.sources = [] - self.fakeartifact.sources.append(coll1) - results = self._RunClientActionArtifact(client_id, client_mock, - ["FakeArtifact"]) - self.assertEmpty(results) - def testRegistryValueArtifact(self): client_id = self.SetupClient(0, system="Linux") - with vfs_test_lib.VFSOverrider(rdf_paths.PathSpec.PathType.REGISTRY, - vfs_test_lib.FakeRegistryVFSHandler): - with vfs_test_lib.VFSOverrider(rdf_paths.PathSpec.PathType.OS, - vfs_test_lib.FakeFullVFSHandler): - + with vfs_test_lib.VFSOverrider( + rdf_paths.PathSpec.PathType.REGISTRY, + vfs_test_lib.FakeRegistryVFSHandler, + ): + with vfs_test_lib.VFSOverrider( + rdf_paths.PathSpec.PathType.OS, vfs_test_lib.FakeFullVFSHandler + ): client_mock = action_mocks.ActionMock(standard.GetFileStat) coll1 = rdf_artifacts.ArtifactSource( type=rdf_artifacts.ArtifactSource.SourceType.REGISTRY_VALUE, attributes={ "key_value_pairs": [{ - "key": (r"HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet" - r"\Control\Session Manager"), - "value": "BootExecute" + "key": ( + r"HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet" + r"\Control\Session Manager" + ), + "value": "BootExecute", }] - }) + }, + ) self.fakeartifact.sources.append(coll1) artifact_list = ["FakeArtifact"] flow_id = flow_test_lib.TestFlowHelper( @@ -343,7 +316,8 @@ def testRegistryValueArtifact(self): client_mock, artifact_list=artifact_list, creator=self.test_username, - client_id=client_id) + client_id=client_id, + ) # Test the statentry got stored. results = flow_test_lib.GetFlowResults(client_id, flow_id) @@ -352,20 +326,23 @@ def testRegistryValueArtifact(self): def testRegistryDefaultValueArtifact(self): client_id = self.SetupClient(0, system="Linux") - with vfs_test_lib.VFSOverrider(rdf_paths.PathSpec.PathType.REGISTRY, - vfs_test_lib.FakeRegistryVFSHandler): - with vfs_test_lib.VFSOverrider(rdf_paths.PathSpec.PathType.OS, - vfs_test_lib.FakeFullVFSHandler): - + with vfs_test_lib.VFSOverrider( + rdf_paths.PathSpec.PathType.REGISTRY, + vfs_test_lib.FakeRegistryVFSHandler, + ): + with vfs_test_lib.VFSOverrider( + rdf_paths.PathSpec.PathType.OS, vfs_test_lib.FakeFullVFSHandler + ): client_mock = action_mocks.ActionMock(standard.GetFileStat) coll1 = rdf_artifacts.ArtifactSource( type=rdf_artifacts.ArtifactSource.SourceType.REGISTRY_VALUE, attributes={ "key_value_pairs": [{ - "key": (r"HKEY_LOCAL_MACHINE/SOFTWARE/ListingTest"), - "value": "" + "key": r"HKEY_LOCAL_MACHINE/SOFTWARE/ListingTest", + "value": "", }] - }) + }, + ) self.fakeartifact.sources.append(coll1) artifact_list = ["FakeArtifact"] flow_id = flow_test_lib.TestFlowHelper( @@ -373,7 +350,8 @@ def testRegistryDefaultValueArtifact(self): client_mock, artifact_list=artifact_list, creator=self.test_username, - client_id=client_id) + client_id=client_id, + ) results = flow_test_lib.GetFlowResults(client_id, flow_id) self.assertIsInstance(results[0], rdf_client_fs.StatEntry) @@ -383,40 +361,50 @@ def testSupportedOS(self): """Test supported_os inside the collector object.""" client_id = self.SetupClient(0, system="Linux") with mock.patch.object(psutil, "process_iter", ProcessIter): - # Run with false condition. client_mock = action_mocks.ActionMock(standard.ListProcesses) coll1 = rdf_artifacts.ArtifactSource( type=rdf_artifacts.ArtifactSource.SourceType.GRR_CLIENT_ACTION, attributes={"client_action": standard.ListProcesses.__name__}, - supported_os=["Windows"]) + supported_os=["Windows"], + ) self.fakeartifact.sources.append(coll1) - results = self._RunClientActionArtifact(client_id, client_mock, - ["FakeArtifact"]) + results = self._RunClientActionArtifact( + client_id, client_mock, ["FakeArtifact"] + ) self.assertEmpty(results) - # Now run with matching or condition. - coll1.conditions = [] coll1.supported_os = ["Linux", "Windows"] self.fakeartifact.sources = [] self.fakeartifact.sources.append(coll1) - results = self._RunClientActionArtifact(client_id, client_mock, - ["FakeArtifact"]) + results = self._RunClientActionArtifact( + client_id, client_mock, ["FakeArtifact"] + ) self.assertTrue(results) - # Now run with impossible or condition. - coll1.conditions = ["os == 'Linux' or os == 'Windows'"] coll1.supported_os = ["NotTrue"] self.fakeartifact.sources = [] self.fakeartifact.sources.append(coll1) - results = self._RunClientActionArtifact(client_id, client_mock, - ["FakeArtifact"]) + results = self._RunClientActionArtifact( + client_id, client_mock, ["FakeArtifact"] + ) self.assertEmpty(results) - def _RunClientActionArtifact(self, - client_id, - client_mock, - artifact_list, - implementation_type=None): + coll1.supported_os = ["Linux", "Windows"] + self.fakeartifact.supported_os = ["Linux"] + results = self._RunClientActionArtifact( + client_id, client_mock, ["FakeArtifact"] + ) + self.assertTrue(results) + + self.fakeartifact.supported_os = ["Windows"] + results = self._RunClientActionArtifact( + client_id, client_mock, ["FakeArtifact"] + ) + self.assertEmpty(results) + + def _RunClientActionArtifact( + self, client_id, client_mock, artifact_list, implementation_type=None + ): self.output_count += 1 flow_id = flow_test_lib.TestFlowHelper( collectors.ArtifactCollectorFlow.__name__, @@ -430,8 +418,11 @@ def _RunClientActionArtifact(self, return flow_test_lib.GetFlowResults(client_id, flow_id) - @mock.patch.object(parsers, "SINGLE_RESPONSE_PARSER_FACTORY", - factory.Factory(parsers.SingleResponseParser)) + @mock.patch.object( + parsers, + "SINGLE_RESPONSE_PARSER_FACTORY", + factory.Factory(parsers.SingleResponseParser), + ) def testParsingFailure(self): """Test a command artifact where parsing the response fails.""" @@ -441,8 +432,9 @@ def testParsingFailure(self): client_id = self.SetupClient(0, system="Linux") - parsers.SINGLE_RESPONSE_PARSER_FACTORY.Register("TestCmd", - TestCmdNullParser) + parsers.SINGLE_RESPONSE_PARSER_FACTORY.Register( + "TestCmd", TestCmdNullParser + ) artifact_list = ["TestUntypedEchoArtifact"] flow_id = flow_test_lib.StartAndRunFlow( @@ -450,12 +442,12 @@ def testParsingFailure(self): action_mocks.ActionMock(standard.ExecuteCommand), artifact_list=artifact_list, apply_parsers=True, - client_id=client_id) + client_id=client_id, + ) results = flow_test_lib.GetFlowResults(client_id, flow_id) self.assertEmpty(results) def testFlowProgressHasEntryForArtifactWithoutResults(self): - client_id = self.SetupClient(0, system="Linux") with mock.patch.object(psutil, "process_iter", lambda: iter([])): client_mock = action_mocks.ActionMock(standard.ListProcesses) @@ -463,13 +455,16 @@ def testFlowProgressHasEntryForArtifactWithoutResults(self): self.fakeartifact.sources.append( rdf_artifacts.ArtifactSource( type=rdf_artifacts.ArtifactSource.SourceType.GRR_CLIENT_ACTION, - attributes={"client_action": standard.ListProcesses.__name__})) + attributes={"client_action": standard.ListProcesses.__name__}, + ) + ) flow_id = flow_test_lib.TestFlowHelper( collectors.ArtifactCollectorFlow.__name__, client_mock, artifact_list=["FakeArtifact"], - client_id=client_id) + client_id=client_id, + ) progress = flow_test_lib.GetFlowProgress(client_id, flow_id) self.assertLen(progress.artifacts, 1) @@ -477,11 +472,10 @@ def testFlowProgressHasEntryForArtifactWithoutResults(self): self.assertEqual(progress.artifacts[0].num_results, 0) def testFlowProgressIsCountingResults(self): - def _Iter(): return iter([ client_test_lib.MockWindowsProcess(), - client_test_lib.MockWindowsProcess() + client_test_lib.MockWindowsProcess(), ]) client_id = self.SetupClient(0, system="Linux") @@ -491,13 +485,16 @@ def _Iter(): self.fakeartifact.sources.append( rdf_artifacts.ArtifactSource( type=rdf_artifacts.ArtifactSource.SourceType.GRR_CLIENT_ACTION, - attributes={"client_action": standard.ListProcesses.__name__})) + attributes={"client_action": standard.ListProcesses.__name__}, + ) + ) flow_id = flow_test_lib.TestFlowHelper( collectors.ArtifactCollectorFlow.__name__, client_mock, artifact_list=["FakeArtifact"], - client_id=client_id) + client_id=client_id, + ) progress = flow_test_lib.GetFlowProgress(client_id, flow_id) self.assertLen(progress.artifacts, 1) @@ -505,19 +502,22 @@ def _Iter(): self.assertEqual(progress.artifacts[0].num_results, 2) def testProcessesResultsOfFailedChildArtifactCollector(self): - client_id = self.SetupClient(0, system="Linux") client_mock = action_mocks.ActionMock(standard.ListProcesses) self.fakeartifact.sources.append( rdf_artifacts.ArtifactSource( type=rdf_artifacts.ArtifactSource.SourceType.ARTIFACT_GROUP, - attributes={"names": ["FakeArtifact2"]})) + attributes={"names": ["FakeArtifact2"]}, + ) + ) self.fakeartifact2.sources.append( rdf_artifacts.ArtifactSource( type=rdf_artifacts.ArtifactSource.SourceType.GRR_CLIENT_ACTION, - attributes={"client_action": standard.ListProcesses.__name__})) + attributes={"client_action": standard.ListProcesses.__name__}, + ) + ) def _RunListProcesses(self, args): self.SendReply(rdf_client.Process(pid=123)) @@ -529,7 +529,8 @@ def _RunListProcesses(self, args): client_mock, artifact_list=["FakeArtifact"], client_id=client_id, - check_flow_errors=False) + check_flow_errors=False, + ) results = flow_test_lib.GetFlowResults(client_id, flow_id) self.assertLen(results, 1) @@ -651,8 +652,9 @@ def GetFileStat(self, _): self.assertNotIn("FileFinderOS", client_mock.called_actions) -class RelationalTestArtifactCollectors(ArtifactCollectorsTestMixin, - test_lib.GRRBaseTest): +class RelationalTestArtifactCollectors( + ArtifactCollectorsTestMixin, test_lib.GRRBaseTest +): def testRunGrrClientActionArtifactSplit(self): """Test that artifacts get split into separate collections.""" @@ -662,7 +664,8 @@ def testRunGrrClientActionArtifactSplit(self): coll1 = rdf_artifacts.ArtifactSource( type=rdf_artifacts.ArtifactSource.SourceType.GRR_CLIENT_ACTION, - attributes={"client_action": standard.ListProcesses.__name__}) + attributes={"client_action": standard.ListProcesses.__name__}, + ) self.fakeartifact.sources.append(coll1) self.fakeartifact2.sources.append(coll1) artifact_list = ["FakeArtifact", "FakeArtifact2"] @@ -672,10 +675,13 @@ def testRunGrrClientActionArtifactSplit(self): artifact_list=artifact_list, creator=self.test_username, client_id=client_id, - split_output_by_artifact=True) + split_output_by_artifact=True, + ) results_by_tag = flow_test_lib.GetFlowResultsByTag(client_id, flow_id) - self.assertCountEqual(results_by_tag.keys(), - ["artifact:FakeArtifact", "artifact:FakeArtifact2"]) + self.assertCountEqual( + results_by_tag.keys(), + ["artifact:FakeArtifact", "artifact:FakeArtifact2"], + ) def testOldClientSnapshotFallbackIfInterpolationFails(self): rel_db = data_store.REL_DB @@ -690,25 +696,29 @@ def testOldClientSnapshotFallbackIfInterpolationFails(self): rdf_client.User(username="user2"), ] snapshot_0 = rdf_objects.ClientSnapshot( - client_id=client_id, knowledge_base=kb_0) + client_id=client_id, knowledge_base=kb_0 + ) rel_db.WriteClientSnapshot(snapshot_0) kb_1 = rdf_client.KnowledgeBase(os="Linux") snapshot_1 = rdf_objects.ClientSnapshot( - client_id=client_id, knowledge_base=kb_1) + client_id=client_id, knowledge_base=kb_1 + ) rel_db.WriteClientSnapshot(snapshot_1) kb_2 = rdf_client.KnowledgeBase(os="Linux") snapshot_2 = rdf_objects.ClientSnapshot( - client_id=client_id, knowledge_base=kb_2) + client_id=client_id, knowledge_base=kb_2 + ) rel_db.WriteClientSnapshot(snapshot_2) with temp.AutoTempDirPath(remove_non_empty=True) as dirpath: - filesystem_test_lib.CreateFile( - os.path.join(dirpath, "user1", "quux", "thud")) + os.path.join(dirpath, "user1", "quux", "thud") + ) filesystem_test_lib.CreateFile( - os.path.join(dirpath, "user2", "quux", "norf")) + os.path.join(dirpath, "user2", "quux", "norf") + ) # Write a fake artifact. path = os.path.join(dirpath, "%%users.username%%", "quux", "*") @@ -720,8 +730,10 @@ def testOldClientSnapshotFallbackIfInterpolationFails(self): type=rdf_artifacts.ArtifactSource.SourceType.PATH, attributes={ "paths": [path], - }), - ]) + }, + ), + ], + ) rel_db.WriteArtifact(art) artifact_registry.REGISTRY.ReloadDatastoreArtifacts() @@ -749,21 +761,23 @@ def testOldClientSnapshotFallbackUsesLatestApplicable(self): # Write some fake snapshot history. kb_0 = rdf_client.KnowledgeBase(os="Linux", os_release="rel0") snapshot_0 = rdf_objects.ClientSnapshot( - client_id=client_id, knowledge_base=kb_0) + client_id=client_id, knowledge_base=kb_0 + ) rel_db.WriteClientSnapshot(snapshot_0) kb_1 = rdf_client.KnowledgeBase(os="Linux", os_release="rel1") snapshot_1 = rdf_objects.ClientSnapshot( - client_id=client_id, knowledge_base=kb_1) + client_id=client_id, knowledge_base=kb_1 + ) rel_db.WriteClientSnapshot(snapshot_1) kb_2 = rdf_client.KnowledgeBase(os="Linux") snapshot_2 = rdf_objects.ClientSnapshot( - client_id=client_id, knowledge_base=kb_2) + client_id=client_id, knowledge_base=kb_2 + ) rel_db.WriteClientSnapshot(snapshot_2) with temp.AutoTempDirPath(remove_non_empty=True) as dirpath: - filesystem_test_lib.CreateFile(os.path.join(dirpath, "rel0", "quux")) filesystem_test_lib.CreateFile(os.path.join(dirpath, "rel1", "norf")) @@ -776,8 +790,10 @@ def testOldClientSnapshotFallbackUsesLatestApplicable(self): type=rdf_artifacts.ArtifactSource.SourceType.PATH, attributes={ "paths": [os.path.join(dirpath, "%%os_release%%", "*")], - }), - ]) + }, + ), + ], + ) rel_db.WriteArtifact(art) artifact_registry.REGISTRY.ReloadDatastoreArtifacts() @@ -801,25 +817,27 @@ def testOldClientSnapshotFallbackUsesLatestApplicable(self): class MeetsConditionsTest(test_lib.GRRBaseTest): """Test the module-level method `MeetsConditions`.""" - def testSourceMeetsConditions(self): + def testSourceMeetsOSConditions(self): """Test we can get a GRR client artifact with conditions.""" knowledge_base = rdf_client.KnowledgeBase() knowledge_base.os = "Windows" - # Run with false condition. + # Run with unsupported OS. source = rdf_artifacts.ArtifactSource( type=rdf_artifacts.ArtifactSource.SourceType.GRR_CLIENT_ACTION, attributes={"client_action": standard.ListProcesses.__name__}, - conditions=["os == 'Linux'"]) - self.assertFalse(collectors.MeetsConditions(knowledge_base, source)) + supported_os=["Linux"], + ) + self.assertFalse(collectors.MeetsOSConditions(knowledge_base, source)) - # Run with matching or condition. + # Run with supported OS. source = rdf_artifacts.ArtifactSource( type=rdf_artifacts.ArtifactSource.SourceType.GRR_CLIENT_ACTION, attributes={"client_action": standard.ListProcesses.__name__}, - conditions=["os == 'Linux' or os == 'Windows'"]) - self.assertTrue(collectors.MeetsConditions(knowledge_base, source)) + supported_os=["Linux", "Windows"], + ) + self.assertTrue(collectors.MeetsOSConditions(knowledge_base, source)) class GetArtifactCollectorArgsTest(test_lib.GRRBaseTest): @@ -831,14 +849,16 @@ def SetOS(self, os_name): def ArtifactCollectorArgs(self, artifact_list, collect_knowledge_base=False): flow_args = rdf_artifacts.ArtifactCollectorFlowArgs( artifact_list=artifact_list, - recollect_knowledge_base=collect_knowledge_base) + recollect_knowledge_base=collect_knowledge_base, + ) return collectors.GetArtifactCollectorArgs(flow_args, self.knowledge_base) def setUp(self): super().setUp() - test_artifacts_file = os.path.join(config.CONFIG["Test.data_dir"], - "artifacts", "test_artifacts.json") + test_artifacts_file = os.path.join( + config.CONFIG["Test.data_dir"], "artifacts", "test_artifacts.json" + ) artifact_registry.REGISTRY.ClearSources() artifact_registry.REGISTRY.ClearRegistry() @@ -904,8 +924,10 @@ def testPrepareMultipleArtifacts(self): """Test we can prepare multiple artifacts of different types.""" artifact_list = [ - "TestFilesArtifact", "DepsWindirRegex", "DepsProvidesMultiple", - "WMIActiveScriptEventConsumer" + "TestFilesArtifact", + "DepsWindirRegex", + "DepsProvidesMultiple", + "WMIActiveScriptEventConsumer", ] self.SetOS("Windows") @@ -928,8 +950,10 @@ def testDuplicationChecks(self): """Test duplicated artifacts are only processed once.""" artifact_list = [ - "TestAggregationArtifact", "TestFilesArtifact", "TestCmdArtifact", - "TestFilesArtifact" + "TestAggregationArtifact", + "TestFilesArtifact", + "TestCmdArtifact", + "TestFilesArtifact", ] self.SetOS("Linux") @@ -948,7 +972,8 @@ def testPrepareArtifactFilesClientArtifactCollectorArgs(self): file_path = os.path.join(self.base_path, "numbers.txt") source = rdf_artifacts.ArtifactSource( type=rdf_artifacts.ArtifactSource.SourceType.FILE, - attributes={"paths": [file_path]}) + attributes={"paths": [file_path]}, + ) artifact_obj = artifact_registry.REGISTRY.GetArtifact("TestFileArtifact") artifact_obj.sources.append(source) @@ -1040,23 +1065,24 @@ def testFlagArtifactFiles(self): class TestCmdParser(parser.CommandParser): - output_types = [rdf_client.SoftwarePackages] supported_artifacts = ["TestEchoArtifact"] def Parse(self, cmd, args, stdout, stderr, return_val, knowledge_base): del cmd, args, stderr, return_val, knowledge_base # Unused - yield rdf_client.SoftwarePackages(packages=[ - rdf_client.SoftwarePackage.Installed( - name="Package", - description=stdout, - version="1", - architecture="amd64"), - ]) + yield rdf_client.SoftwarePackages( + packages=[ + rdf_client.SoftwarePackage.Installed( + name="Package", + description=stdout, + version="1", + architecture="amd64", + ), + ] + ) class TestCmdNullParser(parser.CommandParser): - output_types = [rdf_client.SoftwarePackages] supported_artifacts = ["TestUntypedEchoArtifact"] @@ -1067,7 +1093,6 @@ def Parse(self, cmd, args, stdout, stderr, return_val, knowledge_base): class TestFileParser(parsers.SingleFileParser[rdf_protodict.AttributedDict]): - output_types = [rdf_protodict.AttributedDict] supported_artifacts = ["TestFileArtifact"] @@ -1093,8 +1118,9 @@ def InitGRRWithTestArtifacts(self): artifact_registry.REGISTRY.ClearSources() artifact_registry.REGISTRY.ClearRegistry() - test_artifacts_file = os.path.join(config.CONFIG["Test.data_dir"], - "artifacts", "test_artifacts.json") + test_artifacts_file = os.path.join( + config.CONFIG["Test.data_dir"], "artifacts", "test_artifacts.json" + ) artifact_registry.REGISTRY.AddFileSource(test_artifacts_file) self.addCleanup(artifact_registry.REGISTRY.AddDefaultSources) @@ -1133,18 +1159,20 @@ def _RunFlow(self, flow_cls, action, artifact_list, apply_parsers): artifact_list=artifact_list, creator=self.test_username, apply_parsers=apply_parsers, - client_id=self.client_id) + client_id=self.client_id, + ) return flow_test_lib.GetFlowResults(self.client_id, flow_id) def InitializeTestFileArtifact(self, with_pathspec_attribute=False): file_path = os.path.join(self.base_path, "numbers.txt") source = rdf_artifacts.ArtifactSource( type=rdf_artifacts.ArtifactSource.SourceType.FILE, - attributes={"paths": [file_path]}) + attributes={"paths": [file_path]}, + ) if with_pathspec_attribute: source.attributes = { "paths": [file_path], - "pathspec_attribute": "pathspec" + "pathspec_attribute": "pathspec", } artifact_obj = artifact_registry.REGISTRY.GetArtifact("TestFileArtifact") artifact_obj.sources.append(source) @@ -1154,7 +1182,8 @@ def testClientArtifactCollector(self): """Test artifact collector flow with a single artifact.""" filesystem_test_lib.Command( - "/usr/bin/dpkg", args=["--list"], system="Linux") + "/usr/bin/dpkg", args=["--list"], system="Linux" + ) artifact_list = ["TestCmdArtifact"] @@ -1162,7 +1191,8 @@ def testClientArtifactCollector(self): collectors.ClientArtifactCollector, artifact_collector.ArtifactCollector, artifact_list, - apply_parsers=False) + apply_parsers=False, + ) self.assertLen(results, 1) artifact_response = results[0] @@ -1173,7 +1203,8 @@ def testClientArtifactCollectorWithMultipleArtifacts(self): """Test artifact collector flow with a single artifact.""" filesystem_test_lib.Command( - "/usr/bin/dpkg", args=["--list"], system="Linux") + "/usr/bin/dpkg", args=["--list"], system="Linux" + ) artifact_list = ["TestCmdArtifact", "TestOSAgnostic"] @@ -1181,7 +1212,8 @@ def testClientArtifactCollectorWithMultipleArtifacts(self): collectors.ClientArtifactCollector, artifact_collector.ArtifactCollector, artifact_list, - apply_parsers=False) + apply_parsers=False, + ) self.assertLen(results, 2) artifact_response = results[0] @@ -1197,7 +1229,8 @@ def testLinuxMountCmdArtifact(self): artifact_list = ["LinuxMountCmd"] InitGRRWithTestSources( - self, """ + self, + """ name: LinuxMountCmd doc: Linux output of mount. sources: @@ -1207,7 +1240,8 @@ def testLinuxMountCmdArtifact(self): args: [] labels: [System] supported_os: [Linux] -""") +""", + ) self.assertTrue(artifact_registry.REGISTRY.GetArtifact("LinuxMountCmd")) @@ -1216,7 +1250,8 @@ def testLinuxMountCmdArtifact(self): collectors.ArtifactCollectorFlow, standard.ExecuteCommand, artifact_list, - apply_parsers=False) + apply_parsers=False, + ) expected = expected[0] self.assertIsInstance(expected, rdf_client_action.ExecuteResponse) @@ -1225,7 +1260,8 @@ def testLinuxMountCmdArtifact(self): collectors.ClientArtifactCollector, artifact_collector.ArtifactCollector, artifact_list, - apply_parsers=False) + apply_parsers=False, + ) artifact_response = results[0] self.assertIsInstance(artifact_response, rdf_client_action.ExecuteResponse) @@ -1237,7 +1273,8 @@ def testBasicRegistryKeyArtifact(self): artifact_list = ["TestRegistryKey"] InitGRRWithTestSources( - self, r""" + self, + r""" name: TestRegistryKey doc: A sample registry key artifact. sources: @@ -1246,13 +1283,16 @@ def testBasicRegistryKeyArtifact(self): keys: [ 'HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\Session Manager' ] -""") - - with vfs_test_lib.VFSOverrider(rdf_paths.PathSpec.PathType.REGISTRY, - vfs_test_lib.FakeRegistryVFSHandler): - with vfs_test_lib.VFSOverrider(rdf_paths.PathSpec.PathType.OS, - vfs_test_lib.FakeFullVFSHandler): +""", + ) + with vfs_test_lib.VFSOverrider( + rdf_paths.PathSpec.PathType.REGISTRY, + vfs_test_lib.FakeRegistryVFSHandler, + ): + with vfs_test_lib.VFSOverrider( + rdf_paths.PathSpec.PathType.OS, vfs_test_lib.FakeFullVFSHandler + ): # Run the ArtifactCollector to get the expected result. flow_id = flow_test_lib.TestFlowHelper( collectors.ArtifactCollectorFlow.__name__, @@ -1260,7 +1300,8 @@ def testBasicRegistryKeyArtifact(self): artifact_list=artifact_list, creator=self.test_username, client_id=self.client_id, - apply_parsers=False) + apply_parsers=False, + ) results = flow_test_lib.GetFlowResults(self.client_id, flow_id) expected = results[0] self.assertIsInstance(expected, rdf_client_fs.StatEntry) @@ -1270,7 +1311,8 @@ def testBasicRegistryKeyArtifact(self): collectors.ClientArtifactCollector, artifact_collector.ArtifactCollector, artifact_list, - apply_parsers=False) + apply_parsers=False, + ) artifact_response = cac_results[0] self.assertIsInstance(artifact_response, rdf_client_fs.StatEntry) @@ -1282,7 +1324,8 @@ def testRegistryKeyArtifactWithWildcard(self): artifact_list = ["TestRegistryKey"] InitGRRWithTestSources( - self, r""" + self, + r""" name: TestRegistryKey doc: A sample registry key artifact. sources: @@ -1291,13 +1334,16 @@ def testRegistryKeyArtifactWithWildcard(self): keys: [ 'HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\Session Manager\*' ] -""") - - with vfs_test_lib.VFSOverrider(rdf_paths.PathSpec.PathType.REGISTRY, - vfs_test_lib.FakeRegistryVFSHandler): - with vfs_test_lib.VFSOverrider(rdf_paths.PathSpec.PathType.OS, - vfs_test_lib.FakeFullVFSHandler): +""", + ) + with vfs_test_lib.VFSOverrider( + rdf_paths.PathSpec.PathType.REGISTRY, + vfs_test_lib.FakeRegistryVFSHandler, + ): + with vfs_test_lib.VFSOverrider( + rdf_paths.PathSpec.PathType.OS, vfs_test_lib.FakeFullVFSHandler + ): # Run the ArtifactCollector to get the expected result. flow_id = flow_test_lib.TestFlowHelper( collectors.ArtifactCollectorFlow.__name__, @@ -1305,7 +1351,8 @@ def testRegistryKeyArtifactWithWildcard(self): artifact_list=artifact_list, creator=self.test_username, client_id=self.client_id, - apply_parsers=False) + apply_parsers=False, + ) results = flow_test_lib.GetFlowResults(self.client_id, flow_id) self.assertIsInstance(results[0], rdf_client_fs.StatEntry) @@ -1314,7 +1361,8 @@ def testRegistryKeyArtifactWithWildcard(self): collectors.ClientArtifactCollector, artifact_collector.ArtifactCollector, artifact_list, - apply_parsers=False) + apply_parsers=False, + ) artifact_response = cac_results[0] self.assertIsInstance(artifact_response, rdf_client_fs.StatEntry) @@ -1326,7 +1374,8 @@ def testRegistryKeyArtifactWithPathRecursion(self): artifact_list = ["TestRegistryKey"] InitGRRWithTestSources( - self, r""" + self, + r""" name: TestRegistryKey doc: A sample registry key artifact. sources: @@ -1335,13 +1384,16 @@ def testRegistryKeyArtifactWithPathRecursion(self): keys: [ 'HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\**\Session Manager\*' ] -""") - - with vfs_test_lib.VFSOverrider(rdf_paths.PathSpec.PathType.REGISTRY, - vfs_test_lib.FakeRegistryVFSHandler): - with vfs_test_lib.VFSOverrider(rdf_paths.PathSpec.PathType.OS, - vfs_test_lib.FakeFullVFSHandler): +""", + ) + with vfs_test_lib.VFSOverrider( + rdf_paths.PathSpec.PathType.REGISTRY, + vfs_test_lib.FakeRegistryVFSHandler, + ): + with vfs_test_lib.VFSOverrider( + rdf_paths.PathSpec.PathType.OS, vfs_test_lib.FakeFullVFSHandler + ): # Run the ArtifactCollector to get the expected result. flow_id = flow_test_lib.TestFlowHelper( collectors.ArtifactCollectorFlow.__name__, @@ -1349,7 +1401,8 @@ def testRegistryKeyArtifactWithPathRecursion(self): artifact_list=artifact_list, creator=self.test_username, client_id=self.client_id, - apply_parsers=False) + apply_parsers=False, + ) expected = flow_test_lib.GetFlowResults(self.client_id, flow_id)[0] self.assertIsInstance(expected, rdf_client_fs.StatEntry) @@ -1358,14 +1411,18 @@ def testRegistryKeyArtifactWithPathRecursion(self): collectors.ClientArtifactCollector, artifact_collector.ArtifactCollector, artifact_list, - apply_parsers=False) + apply_parsers=False, + ) artifact_response = results[0] self.assertIsInstance(artifact_response, rdf_client_fs.StatEntry) self.assertEqual(artifact_response, expected) - @mock.patch.object(parsers, "SINGLE_RESPONSE_PARSER_FACTORY", - factory.Factory(parsers.SingleResponseParser)) + @mock.patch.object( + parsers, + "SINGLE_RESPONSE_PARSER_FACTORY", + factory.Factory(parsers.SingleResponseParser), + ) def testCmdArtifactWithParser(self): """Test a command artifact and parsing the response.""" @@ -1380,7 +1437,8 @@ def testCmdArtifactWithParser(self): collectors.ArtifactCollectorFlow, standard.ExecuteCommand, artifact_list, - apply_parsers=True) + apply_parsers=True, + ) self.assertTrue(expected) expected = expected[0] self.assertIsInstance(expected, rdf_client.SoftwarePackages) @@ -1390,15 +1448,19 @@ def testCmdArtifactWithParser(self): collectors.ClientArtifactCollector, artifact_collector.ArtifactCollector, artifact_list, - apply_parsers=True) + apply_parsers=True, + ) self.assertLen(results, 1) artifact_response = results[0] self.assertIsInstance(artifact_response, rdf_client.SoftwarePackages) self.assertEqual(artifact_response, expected) - @mock.patch.object(parsers, "SINGLE_FILE_PARSER_FACTORY", - factory.Factory(parsers.SingleFileParser)) + @mock.patch.object( + parsers, + "SINGLE_FILE_PARSER_FACTORY", + factory.Factory(parsers.SingleFileParser), + ) def testFileArtifactWithParser(self): """Test collecting a file artifact and parsing the response.""" parsers.SINGLE_FILE_PARSER_FACTORY.Register("TestFile", TestFileParser) @@ -1414,7 +1476,8 @@ def testFileArtifactWithParser(self): artifact_list=artifact_list, creator=self.test_username, apply_parsers=True, - client_id=self.client_id) + client_id=self.client_id, + ) results = flow_test_lib.GetFlowResults(self.client_id, flow_id) expected = results[0] self.assertIsInstance(expected, rdf_protodict.AttributedDict) @@ -1426,20 +1489,25 @@ def testFileArtifactWithParser(self): collectors.ClientArtifactCollector, artifact_collector.ArtifactCollector, artifact_list, - apply_parsers=True) + apply_parsers=True, + ) self.assertLen(cac_results, 1) self.assertEqual(results, cac_results) - @mock.patch.object(parsers, "SINGLE_RESPONSE_PARSER_FACTORY", - factory.Factory(parsers.SingleResponseParser)) + @mock.patch.object( + parsers, + "SINGLE_RESPONSE_PARSER_FACTORY", + factory.Factory(parsers.SingleResponseParser), + ) def testParsingFailure(self): """Test a command artifact where parsing the response fails.""" filesystem_test_lib.Command("/bin/echo", args=["1"]) - parsers.SINGLE_RESPONSE_PARSER_FACTORY.Register("TestCmd", - TestCmdNullParser) + parsers.SINGLE_RESPONSE_PARSER_FACTORY.Register( + "TestCmd", TestCmdNullParser + ) artifact_list = ["TestUntypedEchoArtifact"] # Run the ClientArtifactCollector to get the result. @@ -1447,7 +1515,8 @@ def testParsingFailure(self): collectors.ClientArtifactCollector, artifact_collector.ArtifactCollector, artifact_list, - apply_parsers=True) + apply_parsers=True, + ) self.assertEmpty(results) def testAggregatedArtifact(self): @@ -1462,7 +1531,8 @@ def testAggregatedArtifact(self): collectors.ClientArtifactCollector, artifact_collector.ArtifactCollector, artifact_list, - apply_parsers=False) + apply_parsers=False, + ) self.assertLen(results, 2) artifact_response = results[0] @@ -1486,7 +1556,8 @@ def testArtifactFiles(self): artifact_list=artifact_list, creator=self.test_username, apply_parsers=False, - client_id=self.client_id) + client_id=self.client_id, + ) results = flow_test_lib.GetFlowResults(self.client_id, flow_id) expected = results[0] @@ -1497,7 +1568,8 @@ def testArtifactFiles(self): collectors.ClientArtifactCollector, artifact_collector.ArtifactCollector, artifact_list, - apply_parsers=False) + apply_parsers=False, + ) self.assertLen(cac_results, 1) artifact_response = cac_results[0] @@ -1517,7 +1589,8 @@ def testArtifactFilesWithPathspecAttribute(self): artifact_list=artifact_list, creator=self.test_username, apply_parsers=False, - client_id=self.client_id) + client_id=self.client_id, + ) results = flow_test_lib.GetFlowResults(self.client_id, flow_id) expected = results[0] @@ -1528,7 +1601,8 @@ def testArtifactFilesWithPathspecAttribute(self): collectors.ClientArtifactCollector, artifact_collector.ArtifactCollector, artifact_list, - apply_parsers=False) + apply_parsers=False, + ) self.assertLen(cac_results, 1) artifact_response = cac_results[0] @@ -1542,7 +1616,8 @@ def testArtifactWithoutDependency(self): """Test that artifact list without dependencies does not change.""" InitGRRWithTestSources( - self, """ + self, + """ name: Artifact0 doc: An artifact without dependencies. sources: @@ -1551,10 +1626,12 @@ def testArtifactWithoutDependency(self): paths: - '/sample/path' supported_os: [Linux] -""") +""", + ) artifact_arranger = collectors.ArtifactArranger( - os_name="Linux", artifacts_name_list=["Artifact0"]) + os_name="Linux", artifacts_name_list=["Artifact0"] + ) artifact_list = artifact_arranger.GetArtifactsInProperOrder() self.assertEqual(artifact_list, ["Artifact0"]) @@ -1562,7 +1639,8 @@ def testArtifactWithBasicDependency(self): """Test that an artifact providing the dependency is added to the list.""" InitGRRWithTestSources( - self, """ + self, + """ name: Artifact0 doc: An artifact without dependencies. supported_os: [Linux] @@ -1577,10 +1655,12 @@ def testArtifactWithBasicDependency(self): - '/sample/path' - '/%%users.desktop%%/' supported_os: [Linux] -""") +""", + ) artifact_arranger = collectors.ArtifactArranger( - os_name="Linux", artifacts_name_list=["Artifact1"]) + os_name="Linux", artifacts_name_list=["Artifact1"] + ) artifact_list = artifact_arranger.GetArtifactsInProperOrder() self.assertEqual(artifact_list, ["Artifact0", "Artifact1"]) @@ -1588,7 +1668,8 @@ def testArtifactWithDependencyChain(self): """Test an artifact that depends on artifacts with more dependencies.""" InitGRRWithTestSources( - self, """ + self, + """ name: Artifact0 doc: An artifact without dependencies. sources: @@ -1624,13 +1705,16 @@ def testArtifactWithDependencyChain(self): paths: - '/%%os%%/' supported_os: [Linux] -""") +""", + ) artifact_arranger = collectors.ArtifactArranger( - os_name="Linux", artifacts_name_list=["Artifact3"]) + os_name="Linux", artifacts_name_list=["Artifact3"] + ) artifact_list = artifact_arranger.GetArtifactsInProperOrder() - self.assertEqual(artifact_list, - ["Artifact0", "Artifact1", "Artifact2", "Artifact3"]) + self.assertEqual( + artifact_list, ["Artifact0", "Artifact1", "Artifact2", "Artifact3"] + ) def main(argv): diff --git a/grr/server/grr_response_server/flows/general/discovery.py b/grr/server/grr_response_server/flows/general/discovery.py index e6f60537c5..ccfe202fba 100644 --- a/grr/server/grr_response_server/flows/general/discovery.py +++ b/grr/server/grr_response_server/flows/general/discovery.py @@ -59,14 +59,26 @@ def Start(self): self.state.fqdn = None self.state.os = None - # We have a RRG-supported client, so we can use it instead of the Python - # agent to get basic system metadata. + # There are three possible scenarios: # - # Note that `CallRRG` calls have to happen before any of the `CallClient` - # calls: because response processing is sequential, in case there is only - # RRG agent is running, flow executor will wait for responses that are not - # going to come, never processing responses from the RRG agent. + # 1. Only Python agent is supported. + # 2. Only RRG is supported. + # 3. Both RRG and Python agent are supported. + # + # Additionally, for backward compatibility we assume that if RRG is not + # explicitly supported, then GRR is definitely supported. This assumption + # may be revisited in the future. + # + # Anyway, if both agents are supported we need to get metadata about both + # of them. If only one is supported we need to get metadata about that one + # and only that one. It is important not to issue the request to the other + # one as the flow will get stuck awaiting the response to come. if self.rrg_support: + if self.python_agent_support: + self.CallClient( + server_stubs.GetClientInfo, + next_state=self.ClientInfo.__name__, + ) self.CallRRG( action=rrg_pb2.Action.GET_SYSTEM_METADATA, args=rrg_get_system_metadata_pb2.Args(), @@ -160,6 +172,16 @@ def HandleRRGGetSystemMetadata( # just overridden. data_store.REL_DB.WriteClientSnapshot(self.state.client) + # Cloud VM metadata collection is not supported in RRG at the moment but we + # still need it, so we fall back to the Python agent. This is the same call + # that we make in the `Interrogate.Platform` method handler. + if result.type in [rrg_os_pb2.Type.LINUX, rrg_os_pb2.Type.WINDOWS]: + self.CallClient( + server_stubs.GetCloudVMMetadata, + rdf_cloud.BuildCloudMetadataRequests(), + next_state=self.CloudMetadata.__name__, + ) + # We replicate behaviour of Python-based agents: once the operating system # is known, we can start the knowledgebase initialization flow. if self.state.client.knowledge_base.os: @@ -359,20 +381,20 @@ def ClientInfo(self, responses): response = responses.First() - if fleetspeak_utils.IsFleetspeakEnabledClient(self.client_id): - # Fetch labels for the client from Fleetspeak. If Fleetspeak doesn't - # have any labels for the GRR client, fall back to labels reported by - # the client. - fleetspeak_labels = fleetspeak_utils.GetLabelsFromFleetspeak( - self.client_id) - if fleetspeak_labels: - response.labels = fleetspeak_labels - data_store.REL_DB.AddClientLabels( - client_id=self.client_id, owner="GRR", labels=fleetspeak_labels) - else: - FLEETSPEAK_UNLABELED_CLIENTS.Increment() - logging.warning("Failed to get labels for Fleetspeak client %s.", - self.client_id) + # Fetch labels for the client from Fleetspeak. If Fleetspeak doesn't + # have any labels for the GRR client, fall back to labels reported by + # the client. + fleetspeak_labels = fleetspeak_utils.GetLabelsFromFleetspeak(self.client_id) + if fleetspeak_labels: + response.labels = fleetspeak_labels + data_store.REL_DB.AddClientLabels( + client_id=self.client_id, owner="GRR", labels=fleetspeak_labels + ) + else: + FLEETSPEAK_UNLABELED_CLIENTS.Increment() + logging.warning( + "Failed to get labels for Fleetspeak client %s.", self.client_id + ) sanitized_labels = [] for label in response.labels: diff --git a/grr/server/grr_response_server/flows/general/discovery_test.py b/grr/server/grr_response_server/flows/general/discovery_test.py index 0588588f0d..c081cfff58 100644 --- a/grr/server/grr_response_server/flows/general/discovery_test.py +++ b/grr/server/grr_response_server/flows/general/discovery_test.py @@ -16,14 +16,17 @@ from grr_response_core.lib.rdfvalues import client as rdf_client from grr_response_core.lib.rdfvalues import client_action as rdf_client_action from grr_response_core.lib.rdfvalues import client_fs as rdf_client_fs +from grr_response_core.lib.rdfvalues import flows as rdf_flows from grr_response_core.lib.rdfvalues import paths as rdf_paths from grr_response_core.lib.rdfvalues import structs as rdf_structs +from grr_response_server import action_registry from grr_response_server import artifact_registry from grr_response_server import client_index from grr_response_server import data_store from grr_response_server import events from grr_response_server import fleetspeak_utils from grr_response_server import flow_responses +from grr_response_server import server_stubs from grr_response_server.databases import db as abstract_db from grr_response_server.databases import db_test_utils from grr_response_server.flows.general import discovery @@ -37,6 +40,7 @@ from grr.test_lib import stats_test_lib from grr.test_lib import test_lib from grr.test_lib import vfs_test_lib +from grr_response_proto import rrg_pb2 from grr_response_proto.rrg import os_pb2 as rrg_os_pb2 from grr_response_proto.rrg.action import get_system_metadata_pb2 as rrg_get_system_metadata_pb2 @@ -199,7 +203,7 @@ def setUp(self): def _SetupMinimalClient(self): client_id = "C.0000000000000000" - data_store.REL_DB.WriteClientMetadata(client_id, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id) return client_id @@ -348,7 +352,6 @@ def testFleetspeakClient(self, mock_labels_fn): client_id = "C.0000000000000001" data_store.REL_DB.WriteClientMetadata( client_id, - fleetspeak_enabled=True, fleetspeak_validation_info={"IP": "12.34.56.78"}) client_mock = action_mocks.InterrogatedClient() client_mock.InitializeClient( @@ -382,7 +385,7 @@ def testFleetspeakClient(self, mock_labels_fn): def testFleetspeakClient_OnlyGRRLabels(self, mock_labels_fn): mock_labels_fn.return_value = [] client_id = "C.0000000000000001" - data_store.REL_DB.WriteClientMetadata(client_id, fleetspeak_enabled=True) + data_store.REL_DB.WriteClientMetadata(client_id) client_mock = action_mocks.InterrogatedClient() client_mock.InitializeClient( fqdn="fleetspeak.test.com", @@ -484,11 +487,9 @@ def testSourceFlowIdIsSet(self): @db_test_lib.WithDatabase def testHandleRRGGetSystemMetadata(self, db: abstract_db.Database): - client_id = db_test_utils.InitializeClient(db) + client_id = db_test_utils.InitializeRRGClient(db) flow_id = db_test_utils.InitializeFlow(db, client_id) - db.WriteClientMetadata(client_id, rrg_support=True) - result = rrg_get_system_metadata_pb2.Result() result.type = rrg_os_pb2.Type.LINUX result.version = "1.2.3-alpha" @@ -528,6 +529,171 @@ def testHandleRRGGetSystemMetadata(self, db: abstract_db.Database): self.assertEqual(snapshot.knowledge_base.fqdn, "foo.example.com") self.assertEqual(snapshot.os_version, "1.2.3-alpha") + @db_test_lib.WithDatabase + def testHandleRRGGetSystemMetadataCloudVMMetadataLinux( + self, + db: abstract_db.Database, + ): + client_id = db_test_utils.InitializeRRGClient(db) + flow_id = db_test_utils.InitializeFlow(db, client_id) + + result = rrg_get_system_metadata_pb2.Result(type=rrg_os_pb2.Type.LINUX) + result_response = rdf_flow_objects.FlowResponse() + result_response.any_payload = rdf_structs.AnyValue.PackProto2(result) + + status_response = rdf_flow_objects.FlowStatus() + status_response.status = rdf_flow_objects.FlowStatus.Status.OK + + responses = flow_responses.Responses.FromResponsesProto2Any([ + result_response, + status_response, + ]) + + flow_args = discovery.InterrogateArgs() + flow_args.lightweight = False + + rdf_flow = rdf_flow_objects.Flow() + rdf_flow.client_id = client_id + rdf_flow.flow_id = flow_id + rdf_flow.flow_class_name = discovery.Interrogate.__name__ + rdf_flow.args = flow_args + + flow = discovery.Interrogate(rdf_flow) + flow.Start() + flow.HandleRRGGetSystemMetadata(responses) + + # We should collect VM metadata for Linux. + self.assertTrue( + _HasClientActionRequest(flow, server_stubs.GetCloudVMMetadata) + ) + + @db_test_lib.WithDatabase + def testHandleRRGGetSystemMetadataCloudVMMetadataMacOS( + self, + db: abstract_db.Database, + ): + client_id = db_test_utils.InitializeRRGClient(db) + flow_id = db_test_utils.InitializeFlow(db, client_id) + + result = rrg_get_system_metadata_pb2.Result(type=rrg_os_pb2.Type.MACOS) + result_response = rdf_flow_objects.FlowResponse() + result_response.any_payload = rdf_structs.AnyValue.PackProto2(result) + + status_response = rdf_flow_objects.FlowStatus() + status_response.status = rdf_flow_objects.FlowStatus.Status.OK + + responses = flow_responses.Responses.FromResponsesProto2Any([ + result_response, + status_response, + ]) + + flow_args = discovery.InterrogateArgs() + flow_args.lightweight = False + + rdf_flow = rdf_flow_objects.Flow() + rdf_flow.client_id = client_id + rdf_flow.flow_id = flow_id + rdf_flow.flow_class_name = discovery.Interrogate.__name__ + rdf_flow.args = flow_args + + flow = discovery.Interrogate(rdf_flow) + flow.Start() + flow.HandleRRGGetSystemMetadata(responses) + + # We should not collect VM metadata for macOS. + self.assertFalse( + _HasClientActionRequest(flow, server_stubs.GetCloudVMMetadata) + ) + + @db_test_lib.WithDatabase + def testStartRRGOnly(self, db: abstract_db.Database): + client_id = db_test_utils.InitializeRRGClient(db) + flow_id = db_test_utils.InitializeFlow(db, client_id) + + flow_args = discovery.InterrogateArgs() + flow_args.lightweight = False + + rdf_flow = rdf_flow_objects.Flow() + rdf_flow.client_id = client_id + rdf_flow.flow_id = flow_id + rdf_flow.flow_class_name = discovery.Interrogate.__name__ + rdf_flow.args = flow_args + + flow = discovery.Interrogate(rdf_flow) + flow.Start() + + self.assertFalse(_HasClientActionRequest(flow, server_stubs.GetClientInfo)) + self.assertTrue(_HasRRGRequest(flow, rrg_pb2.Action.GET_SYSTEM_METADATA)) + + @db_test_lib.WithDatabase + def testStartPythonAgent(self, db: abstract_db.Database): + client_id = db_test_utils.InitializeClient(db) + flow_id = db_test_utils.InitializeFlow(db, client_id) + + flow_args = discovery.InterrogateArgs() + flow_args.lightweight = False + + rdf_flow = rdf_flow_objects.Flow() + rdf_flow.client_id = client_id + rdf_flow.flow_id = flow_id + rdf_flow.flow_class_name = discovery.Interrogate.__name__ + rdf_flow.args = flow_args + + flow = discovery.Interrogate(rdf_flow) + flow.Start() + + self.assertTrue(_HasClientActionRequest(flow, server_stubs.GetClientInfo)) + self.assertFalse(_HasRRGRequest(flow, rrg_pb2.Action.GET_SYSTEM_METADATA)) + + @db_test_lib.WithDatabase + def testStartBothAgents(self, db: abstract_db.Database): + client_id = db_test_utils.InitializeRRGClient(db) + flow_id = db_test_utils.InitializeFlow(db, client_id) + + startup = rdf_client.StartupInfo() + startup.client_info.client_version = 4321 + db.WriteClientStartupInfo(client_id, startup) + + flow_args = discovery.InterrogateArgs() + flow_args.lightweight = False + + rdf_flow = rdf_flow_objects.Flow() + rdf_flow.client_id = client_id + rdf_flow.flow_id = flow_id + rdf_flow.flow_class_name = discovery.Interrogate.__name__ + rdf_flow.args = flow_args + + flow = discovery.Interrogate(rdf_flow) + flow.Start() + + self.assertTrue(_HasClientActionRequest(flow, server_stubs.GetClientInfo)) + self.assertTrue(_HasRRGRequest(flow, rrg_pb2.Action.GET_SYSTEM_METADATA)) + + +def _HasClientActionRequest( + flow: discovery.Interrogate, + action: type[server_stubs.ClientActionStub], +) -> bool: + """Checks whether the given flow has a request for the given action.""" + action_id = action_registry.ID_BY_ACTION_STUB[action] + + def IsAction(request: rdf_flows.ClientActionRequest) -> bool: + return request.action_identifier == action_id + + return any(map(IsAction, flow.client_action_requests)) + + +def _HasRRGRequest( + flow: discovery.Interrogate, + action: rrg_pb2.Action, +) -> bool: + """Checks whether the given flow has a request for the given RRG action.""" + + def IsAction(request: rrg_pb2.Request) -> bool: + return request.action == action + + return any(map(IsAction, flow.rrg_requests)) + def main(argv): # Run the full test suite diff --git a/grr/server/grr_response_server/flows/general/export.py b/grr/server/grr_response_server/flows/general/export.py index ad0252ff9d..8f434bbc6f 100644 --- a/grr/server/grr_response_server/flows/general/export.py +++ b/grr/server/grr_response_server/flows/general/export.py @@ -36,8 +36,6 @@ def CollectionItemToClientPath(item, client_id=None): return db.ClientPath.FromPathSpec(client_id, item.pathspec) elif isinstance(item, rdf_file_finder.FileFinderResult): return db.ClientPath.FromPathSpec(client_id, item.stat_entry.pathspec) - elif isinstance(item, rdf_file_finder.CollectSingleFileResult): - return db.ClientPath.FromPathSpec(client_id, item.stat.pathspec) elif isinstance(item, rdf_file_finder.CollectMultipleFilesResult): return db.ClientPath.FromPathSpec(client_id, item.stat.pathspec) elif isinstance(item, rdf_file_finder.CollectFilesByKnownPathResult): diff --git a/grr/server/grr_response_server/flows/general/file_finder.py b/grr/server/grr_response_server/flows/general/file_finder.py index 646ef10f0d..43d29aeb74 100644 --- a/grr/server/grr_response_server/flows/general/file_finder.py +++ b/grr/server/grr_response_server/flows/general/file_finder.py @@ -2,10 +2,10 @@ """Search for certain files, filter them by given criteria and do something.""" import stat -from typing import Optional -from typing import Sequence +from typing import Collection, Optional, Sequence, Set, Tuple from grr_response_core.lib import artifact_utils +from grr_response_core.lib import rdfvalue from grr_response_core.lib.rdfvalues import client_action as rdf_client_action from grr_response_core.lib.rdfvalues import client_fs as rdf_client_fs from grr_response_core.lib.rdfvalues import file_finder as rdf_file_finder @@ -14,6 +14,7 @@ from grr_response_server import data_store from grr_response_server import file_store from grr_response_server import flow_base +from grr_response_server import flow_responses from grr_response_server import server_stubs from grr_response_server.databases import db from grr_response_server.flows.general import filesystem @@ -392,6 +393,51 @@ def End(self, responses): self.Log("Found and processed %d files.", self.state.files_found) +def _GetPendingBlobIDs( + responses: Collection[rdf_file_finder.FileFinderResult], +) -> Sequence[Tuple[rdf_file_finder.FileFinderResult, Set[rdf_objects.BlobID]]]: + """For each FileFinderResult get reported but not yet stored blobs. + + Args: + responses: A collection of FileFinderResults containing transferred file + chunks. + + Returns: + A sequence of tuples (). + Even though returning a dict would be more correct conceptually, this + is not possible as FileFinderResult is not hashable and can't be used + as a key. + """ + response_blob_ids = {} + blob_id_responses = {} + blob_ids = set() + for idx, r in enumerate(responses): + # Store the total number of chunks per response. + response_blob_ids[idx] = set() + for c in r.transferred_file.chunks: + blob_id = rdf_objects.BlobID.FromSerializedBytes(c.digest) + blob_ids.add(blob_id) + response_blob_ids[idx].add(blob_id) + + # For each blob store a set of indexes of responses that have it. + # Note that the same blob may be present in more than one response + # (blobs are just data). + blob_id_responses.setdefault(blob_id, set()).add(idx) + + blobs_present = data_store.BLOBS.CheckBlobsExist(blob_ids) + for blob_id, is_present in blobs_present.items(): + if not is_present: + continue + + # If the blob is present, decrement counters for relevant responses. + for response_idx in blob_id_responses[blob_id]: + response_blob_ids[response_idx].remove(blob_id) + + return [ + (responses[idx], blob_ids) for idx, blob_ids in response_blob_ids.items() + ] + + class ClientFileFinder(flow_base.FlowBase): """A client side file finder flow.""" @@ -400,6 +446,9 @@ class ClientFileFinder(flow_base.FlowBase): args_type = rdf_file_finder.FileFinderArgs behaviours = flow_base.BEHAVIOUR_BASIC + BLOB_CHECK_DELAY = rdfvalue.Duration("60s") + MAX_BLOB_CHECKS = 60 + def Start(self): """Issue the find request.""" super().Start() @@ -415,7 +464,10 @@ def Start(self): self.CallClient( stub, request=interpolated_args, - next_state=self.StoreResults.__name__) + next_state=self.StoreResultsWithoutBlobs.__name__, + ) + + self.state.num_blob_waits = 0 def _InterpolatePaths(self, globs: Sequence[str]) -> Optional[Sequence[str]]: kb = self.client_knowledge_base @@ -447,25 +499,58 @@ def _InterpolatePaths(self, globs: Sequence[str]) -> Optional[Sequence[str]]: return paths - def StoreResults(self, responses): + def StoreResultsWithoutBlobs( + self, + responses: flow_responses.Responses[rdf_file_finder.FileFinderResult], + ) -> None: """Stores the results returned by the client to the db.""" if not responses.success: raise flow_base.FlowError(responses.status) self.state.files_found = len(responses) transferred_file_responses = [] - stat_entries = [] + stat_entry_responses = [] + # Split the responses into the ones that just contain file stats + # and the ones actually referencing uploaded chunks. for response in responses: if response.HasField("transferred_file"): transferred_file_responses.append(response) elif response.HasField("stat_entry"): - stat_entries.append(response.stat_entry) + stat_entry_responses.append(response) - client_path_hash_id = self._WriteFilesContent(transferred_file_responses) + self._WriteStatEntries([r.stat_entry for r in stat_entry_responses]) + for r in stat_entry_responses: + self.SendReply(r) - self._WriteStatEntries(stat_entries) + if transferred_file_responses: + self.CallStateInline( + next_state=self.StoreResultsWithBlobs.__name__, + messages=transferred_file_responses, + ) - for response in responses: + def StoreResultsWithBlobs( + self, + responses: flow_responses.Responses[rdf_file_finder.FileFinderResult], + ) -> None: + """Stores the results returned by the client to the db.""" + complete_responses = [] + incomplete_responses = [] + + response_pending_blob_ids = _GetPendingBlobIDs(list(responses)) + # Needed in case we need to report an error (see below). + sample_pending_blob_id = None + num_pending_blobs = 0 + for response, pending_blob_ids in response_pending_blob_ids: + if not pending_blob_ids: + complete_responses.append(response) + else: + incomplete_responses.append(response) + sample_pending_blob_id = list(pending_blob_ids)[0] + num_pending_blobs += len(pending_blob_ids) + + client_path_hash_id = self._WriteFilesContent(complete_responses) + + for response in complete_responses: pathspec = response.stat_entry.pathspec client_path = db.ClientPath.FromPathSpec(self.client_id, pathspec) @@ -484,25 +569,52 @@ def StoreResults(self, responses): self.SendReply(response) + if incomplete_responses: + self.state.num_blob_waits += 1 + + self.Log( + "Waiting for blobs to be written to the blob store. Iteration: %d out" + " of %d. Blobs pending: %d", + self.state.num_blob_waits, + self.MAX_BLOB_CHECKS, + num_pending_blobs, + ) + + if self.state.num_blob_waits > self.MAX_BLOB_CHECKS: + self.Error( + "Could not find one of referenced blobs " + f"(sample id: {sample_pending_blob_id}). " + "This is a sign of datastore inconsistency." + ) + return + + start_time = rdfvalue.RDFDatetime.Now() + self.BLOB_CHECK_DELAY + self.CallState( + next_state=self.StoreResultsWithBlobs.__name__, + responses=incomplete_responses, + start_time=start_time, + ) + def _WriteFilesContent( self, - responses: list[rdf_file_finder.FileFinderResult], + complete_responses: list[rdf_file_finder.FileFinderResult], ) -> dict[db.ClientPath, rdf_objects.SHA256HashID]: """Writes file contents of multiple files to the relational database. Args: - responses: A list of file finder results to write to the file store. + complete_responses: A list of file finder results to write to the file + store. Returns: - A mapping from paths to the SHA-256 hashes of the files written to the - file store. + A mapping from paths to the SHA-256 hashes of the files written + to the file store. """ client_path_blob_refs = dict() client_path_path_info = dict() client_path_hash_id = dict() client_path_sizes = dict() - for response in responses: + for response in complete_responses: path_info = rdf_objects.PathInfo.FromStatEntry(response.stat_entry) chunks = response.transferred_file.chunks diff --git a/grr/server/grr_response_server/flows/general/file_finder_test.py b/grr/server/grr_response_server/flows/general/file_finder_test.py index 891a929031..5d9db53784 100644 --- a/grr/server/grr_response_server/flows/general/file_finder_test.py +++ b/grr/server/grr_response_server/flows/general/file_finder_test.py @@ -7,7 +7,7 @@ import os import stat import struct -from typing import List, Optional, Sequence, Any +from typing import Any, List, Optional, Sequence from unittest import mock from absl import app @@ -26,6 +26,7 @@ from grr_response_server.databases import db from grr_response_server.databases import db_test_utils from grr_response_server.flows.general import file_finder +from grr_response_server.flows.general import transfer from grr_response_server.rdfvalues import objects as rdf_objects from grr.test_lib import action_mocks from grr.test_lib import filesystem_test_lib @@ -1039,6 +1040,42 @@ def testClientFileFinder(self): "parser_test/InstallHistory.plist" ]) + @mock.patch.object( + file_finder.ClientFileFinder, "BLOB_CHECK_DELAY", rdfvalue.Duration(0) + ) + def testErrorsWhenDownloadingFileAndNotReceivingBlobs(self): + paths = [os.path.join(self.base_path, "test.plist")] + action = rdf_file_finder.FileFinderAction( + action_type=rdf_file_finder.FileFinderAction.Action.DOWNLOAD + ) + + # Make sure BlobHandler doesn't accept any incoming blobs. Thus + # ClientFileFinder will timeout waiting for one. + with mock.patch.object(transfer.BlobHandler, "ProcessMessages"): + flow_id = flow_test_lib.StartFlow( + file_finder.ClientFileFinder, + client_id=self.client_id, + paths=paths, + pathtype=rdf_paths.PathSpec.PathType.OS, + action=action, + creator=self.test_username, + ) + with self.assertRaisesRegex( + RuntimeError, "Could not find one of referenced blobs" + ): + flow_test_lib.RunFlow( + client_id=self.client_id, + flow_id=flow_id, + client_mock=action_mocks.ClientFileFinderClientMock(), + ) + + flow_state = flow_test_lib.GetFlowState(self.client_id, flow_id) + # Check that the flow has actually reached the maximum number of checks. + self.assertEqual( + flow_state.num_blob_waits, + file_finder.ClientFileFinder.MAX_BLOB_CHECKS + 1, + ) + def testUseExternalStores(self): paths = [os.path.join(self.base_path, "test.plist")] action = rdf_file_finder.FileFinderAction( diff --git a/grr/server/grr_response_server/flows/general/filesystem_test.py b/grr/server/grr_response_server/flows/general/filesystem_test.py index d8429b5c2d..bc61aeef1d 100644 --- a/grr/server/grr_response_server/flows/general/filesystem_test.py +++ b/grr/server/grr_response_server/flows/general/filesystem_test.py @@ -972,11 +972,9 @@ class ListDirectoryTest(absltest.TestCase): @db_test_lib.WithDatabase def testHandleRRGGetFileMetadata(self, rel_db: db.Database): - client_id = db_test_utils.InitializeClient(rel_db) + client_id = db_test_utils.InitializeRRGClient(rel_db) flow_id = db_test_utils.InitializeFlow(rel_db, client_id) - rel_db.WriteClientMetadata(client_id, rrg_support=True) - args = filesystem.ListDirectoryArgs() args.pathspec.path = "/foo/bar/baz" diff --git a/grr/server/grr_response_server/flows/general/memory_test.py b/grr/server/grr_response_server/flows/general/memory_test.py index 2e5e0f3db8..c90374eb77 100644 --- a/grr/server/grr_response_server/flows/general/memory_test.py +++ b/grr/server/grr_response_server/flows/general/memory_test.py @@ -1026,7 +1026,6 @@ def setUp(self): # Tracking of time works differently in unprivileged mode. # (There isn't one call to RDFDatetime.Now() per chunk due to batching). - def testScanTimingInformation(self): with test_lib.FakeTime(10000, increment=1): _, _, misses = self._RunYaraProcessScan( @@ -1036,7 +1035,7 @@ def testScanTimingInformation(self): miss = misses[0] self.assertEqual(miss.scan_time_us, 3 * 1e6) - with test_lib.FakeTime(10000, increment=1): + with test_lib.FakeTime(20000, increment=1): matches, _, _ = self._RunYaraProcessScan(self.procs, pids=[102]) self.assertLen(matches, 1) diff --git a/grr/server/grr_response_server/flows/general/registry_init.py b/grr/server/grr_response_server/flows/general/registry_init.py index 70fd1759f1..1e9f039b45 100644 --- a/grr/server/grr_response_server/flows/general/registry_init.py +++ b/grr/server/grr_response_server/flows/general/registry_init.py @@ -7,7 +7,6 @@ from grr_response_server.flows.general import administrative from grr_response_server.flows.general import apple_firmware from grr_response_server.flows.general import artifact_fallbacks -from grr_response_server.flows.general import ca_enroller from grr_response_server.flows.general import collectors from grr_response_server.flows.general import discovery from grr_response_server.flows.general import export diff --git a/grr/server/grr_response_server/flows/general/timeline_test.py b/grr/server/grr_response_server/flows/general/timeline_test.py index 512c3efeda..ccf0d7f2ec 100644 --- a/grr/server/grr_response_server/flows/general/timeline_test.py +++ b/grr/server/grr_response_server/flows/general/timeline_test.py @@ -137,7 +137,7 @@ def testProgress(self): @db_test_lib.WithDatabase def testLogsWarningIfBtimeNotSupported(self, db: abstract_db.Database): client_id = self.client_id - db.WriteClientMetadata(client_id, fleetspeak_enabled=True) + db.WriteClientMetadata(client_id) snapshot = rdf_objects.ClientSnapshot() snapshot.client_id = client_id @@ -164,7 +164,7 @@ def testLogsWarningIfBtimeNotSupported(self, db: abstract_db.Database): @db_test_lib.WithDatabase def testNoLogsIfBtimeSupported(self, db: abstract_db.Database): client_id = self.client_id - db.WriteClientMetadata(client_id, fleetspeak_enabled=True) + db.WriteClientMetadata(client_id) snapshot = rdf_objects.ClientSnapshot() snapshot.client_id = client_id diff --git a/grr/server/grr_response_server/flows/general/transfer.py b/grr/server/grr_response_server/flows/general/transfer.py index b10abb6500..e384d4ac85 100644 --- a/grr/server/grr_response_server/flows/general/transfer.py +++ b/grr/server/grr_response_server/flows/general/transfer.py @@ -116,10 +116,7 @@ def HandleGetFileMetadata( stat_entry.st_mtime = result.metadata.modification_time.seconds stat_entry.st_btime = result.metadata.creation_time.seconds - path_info = rdf_objects.PathInfo.FromPathSpec(self.args.pathspec) - path_info.stat_entry = stat_entry - - self.state.path_info = path_info + self.state.path_info = rdf_objects.PathInfo.FromStatEntry(stat_entry) args = rrg_get_file_contents_pb2.Args() args.path.CopyFrom(result.path) @@ -152,15 +149,16 @@ def HandleGetFileContents( blob_refs.append(blob_ref) - client_path = db.ClientPath.FromPathSpec(self.client_id, self.args.pathspec) + path_info = self.state.path_info + + client_path = db.ClientPath.FromPathInfo(self.client_id, path_info) hash_id = file_store.AddFileWithUnknownHash(client_path, blob_refs) - path_info = self.state.path_info path_info.hash_entry.sha256 = hash_id.AsBytes() path_info.hash_entry.num_bytes = sum(_.size for _ in blob_refs) data_store.REL_DB.WritePathInfos(self.client_id, [path_info]) - self.SendReply(self.state.path_info.stat_entry) + self.SendReply(path_info.stat_entry) class GetFile(flow_base.FlowBase): @@ -502,7 +500,7 @@ def _TryToStartNextPathspec(self): # First stat the file, then hash the file if needed. self._ScheduleStatFile(index, pathspec) - if getattr(self.state, "stop_at_stat", False): + if self.state.stop_at_stat: return self._ScheduleHashFile(index, pathspec) @@ -663,7 +661,7 @@ def _ReceiveFileStat(self, responses): request_data = self.state.request_data_list[index] self.ReceiveFetchedFileStat(stat_entry, request_data) - if getattr(self.state, "stop_at_stat", False): + if self.state.stop_at_stat: self._RemoveCompletedPathspec(index) return @@ -733,7 +731,7 @@ def _ReceiveFileHash(self, responses): request_data = self.state.request_data_list[index] self.ReceiveFetchedFileHash(stat_entry, hash_obj, request_data) - if getattr(self.state, "stop_at_hash", False): + if self.state.stop_at_hash: self._RemoveCompletedPathspec(index) return diff --git a/grr/server/grr_response_server/flows/general/transfer_test.py b/grr/server/grr_response_server/flows/general/transfer_test.py index 7159573432..9309afb193 100644 --- a/grr/server/grr_response_server/flows/general/transfer_test.py +++ b/grr/server/grr_response_server/flows/general/transfer_test.py @@ -1297,11 +1297,9 @@ class GetFileThroughRRGTest(absltest.TestCase): @db_test_lib.WithDatabase def testHandleGetFileMetadata(self, rel_db: db.Database): - client_id = db_test_utils.InitializeClient(rel_db) + client_id = db_test_utils.InitializeRRGClient(rel_db) flow_id = db_test_utils.InitializeFlow(rel_db, client_id) - rel_db.WriteClientMetadata(client_id, rrg_support=True) - args = transfer.GetFileThroughRRG.GetDefaultArgs() args.pathspec.path = "/foo/bar/baz" @@ -1343,11 +1341,9 @@ def testHandleGetFileContents( rel_db: db.Database, bs: blob_store.BlobStore, ) -> None: - client_id = db_test_utils.InitializeClient(rel_db) + client_id = db_test_utils.InitializeRRGClient(rel_db) flow_id = db_test_utils.InitializeFlow(rel_db, client_id) - rel_db.WriteClientMetadata(client_id, rrg_support=True) - blob_data = os.urandom(1337) bs.WriteBlobWithUnknownHash(blob_data) diff --git a/grr/server/grr_response_server/frontend_lib.py b/grr/server/grr_response_server/frontend_lib.py index 978df66861..f6424f500e 100644 --- a/grr/server/grr_response_server/frontend_lib.py +++ b/grr/server/grr_response_server/frontend_lib.py @@ -2,19 +2,16 @@ """The GRR frontend server.""" import logging import time - -from typing import Sequence +from typing import Optional, Sequence from grr_response_core.lib import queues from grr_response_core.lib import rdfvalue from grr_response_core.lib.rdfvalues import client as rdf_client -from grr_response_core.lib.rdfvalues import client_network as rdf_client_network from grr_response_core.lib.rdfvalues import flows as rdf_flows from grr_response_core.lib.rdfvalues import structs as rdf_structs from grr_response_core.lib.util import collection from grr_response_core.lib.util import random from grr_response_core.stats import metrics -from grr_response_server import communicator from grr_response_server import data_store from grr_response_server import events from grr_response_server import message_handlers @@ -27,28 +24,6 @@ from grr_response_proto import rrg_pb2 -CLIENT_PINGS_BY_LABEL = metrics.Counter( - "client_pings_by_label", fields=[("label", str)]) -FRONTEND_ACTIVE_COUNT = metrics.Gauge( - "frontend_active_count", int, fields=[("source", str)]) -FRONTEND_MAX_ACTIVE_COUNT = metrics.Gauge("frontend_max_active_count", int) -FRONTEND_HTTP_REQUESTS = metrics.Counter( - "frontend_http_requests", fields=[("action", str), ("protocol", str)]) -FRONTEND_IN_BYTES = metrics.Counter( - "frontend_in_bytes", fields=[("source", str)]) -FRONTEND_OUT_BYTES = metrics.Counter( - "frontend_out_bytes", fields=[("source", str)]) -FRONTEND_REQUEST_COUNT = metrics.Counter( - "frontend_request_count", fields=[("source", str)]) -FRONTEND_INACTIVE_REQUEST_COUNT = metrics.Counter( - "frontend_inactive_request_count", fields=[("source", str)]) -FRONTEND_REQUEST_LATENCY = metrics.Event( - "frontend_request_latency", fields=[("source", str)]) -GRR_FRONTENDSERVER_HANDLE_TIME = metrics.Event("grr_frontendserver_handle_time") -GRR_FRONTENDSERVER_HANDLE_NUM = metrics.Counter("grr_frontendserver_handle_num") -GRR_MESSAGES_SENT = metrics.Counter("grr_messages_sent") -GRR_UNIQUE_CLIENTS = metrics.Counter("grr_unique_clients") - RRG_PARCEL_COUNT = metrics.Counter( name="rrg_parcel_count", fields=[("sink", str)], @@ -61,116 +36,6 @@ FRONTEND_USERNAME = "GRRFrontEnd" -class ServerCommunicator(communicator.Communicator): - """A communicator which stores certificates using the relational db.""" - - def __init__(self, certificate, private_key): - super().__init__(certificate=certificate, private_key=private_key) - self.common_name = self.certificate.GetCN() - - def _GetRemotePublicKey(self, common_name): - remote_client_id = common_name.Basename() - - try: - md = data_store.REL_DB.ReadClientMetadata(remote_client_id) - except db.UnknownClientError: - GRR_UNIQUE_CLIENTS.Increment() - raise communicator.UnknownClientCertError("Cert not found") - - cert = md.certificate - if cert is None: - raise communicator.UnknownClientCertError("Cert not found") - - if rdfvalue.RDFURN(cert.GetCN()) != rdfvalue.RDFURN(common_name): - logging.error("Stored cert mismatch for %s", common_name) - raise communicator.UnknownClientCertError("Stored cert mismatch") - - return cert.GetPublicKey() - - def VerifyMessageSignature(self, response_comms, packed_message_list, cipher, - cipher_verified, api_version, remote_public_key): - """Verifies the message list signature. - - In the server we check that the timestamp is later than the ping timestamp - stored with the client. This ensures that client responses can not be - replayed. - - Args: - response_comms: The raw response_comms rdfvalue. - packed_message_list: The PackedMessageList rdfvalue from the server. - cipher: The cipher object that should be used to verify the message. - cipher_verified: If True, the cipher's signature is not verified again. - api_version: The api version we should use. - remote_public_key: The public key of the source. - - Returns: - An rdf_flows.GrrMessage.AuthorizationState. - """ - if (not cipher_verified and - not cipher.VerifyCipherSignature(remote_public_key)): - communicator.GRR_UNAUTHENTICATED_MESSAGES.Increment() - return rdf_flows.GrrMessage.AuthorizationState.UNAUTHENTICATED - - try: - client_id = cipher.cipher_metadata.source.Basename() - metadata = data_store.REL_DB.ReadClientMetadata(client_id) - client_time = packed_message_list.timestamp or rdfvalue.RDFDatetime(0) - update_metadata = True - - # This used to be a strict check here so absolutely no out of - # order messages would be accepted ever. Turns out that some - # proxies can send your request with some delay even if the - # client has already timed out (and sent another request in - # the meantime, making the first one out of order). In that - # case we would just kill the whole flow as a - # precaution. Given the behavior of those proxies, this seems - # now excessive and we have changed the replay protection to - # only trigger on messages that are more than one hour old. - if metadata and metadata.clock: - stored_client_time = metadata.clock - - if client_time < stored_client_time - rdfvalue.Duration.From( - 1, rdfvalue.HOURS): - logging.warning("Message desynchronized for %s: %s >= %s", client_id, - stored_client_time, client_time) - # This is likely an old message - return rdf_flows.GrrMessage.AuthorizationState.DESYNCHRONIZED - - # Update the client and server timestamps only if the client time moves - # forward. - if client_time < stored_client_time: - logging.warning("Out of order message for %s: %s > %s", client_id, - stored_client_time, client_time) - update_metadata = False - - communicator.GRR_AUTHENTICATED_MESSAGES.Increment() - - for label in data_store.REL_DB.ReadClientLabels(client_id): - CLIENT_PINGS_BY_LABEL.Increment(fields=[label.name]) - - if not update_metadata: - return rdf_flows.GrrMessage.AuthorizationState.AUTHENTICATED - - source_ip = response_comms.orig_request.source_ip - if source_ip: - last_ip = rdf_client_network.NetworkAddress( - human_readable_address=response_comms.orig_request.source_ip) - else: - last_ip = None - - data_store.REL_DB.WriteClientMetadata( - client_id, - last_ip=last_ip, - last_clock=client_time, - last_ping=rdfvalue.RDFDatetime.Now(), - fleetspeak_enabled=False) - - except communicator.UnknownClientCertError: - pass - - return rdf_flows.GrrMessage.AuthorizationState.AUTHENTICATED - - class FrontEndServer(object): """This is the front end server. @@ -186,14 +51,9 @@ class FrontEndServer(object): """ def __init__(self, - certificate, - private_key, max_queue_size=50, message_expiry_time=120, max_retransmission_time=10): - self._communicator = ServerCommunicator( - certificate=certificate, private_key=private_key) - self.message_expiry_time = message_expiry_time self.max_retransmission_time = max_retransmission_time self.max_queue_size = max_queue_size @@ -203,108 +63,23 @@ def __init__(self, self.unauth_allowed_session_id = rdfvalue.SessionID( queue=queues.ENROLLMENT, flow_name="Enrol") - @GRR_FRONTENDSERVER_HANDLE_NUM.Counted() - @GRR_FRONTENDSERVER_HANDLE_TIME.Timed() - def HandleMessageBundles(self, request_comms, response_comms): - """Processes a queue of messages as passed from the client. - - We basically dispatch all the GrrMessages in the queue to the task scheduler - for backend processing. We then retrieve from the TS the messages destined - for this client. - - Args: - request_comms: A ClientCommunication rdfvalue with messages sent by the - client. source should be set to the client CN. - response_comms: A ClientCommunication rdfvalue of jobs destined to this - client. - - Returns: - tuple of (source, message_count) where message_count is the number of - messages received from the client with common name source. - """ - messages, source, timestamp = self._communicator.DecodeMessages( - request_comms) - - now = time.time() - if messages: - # Receive messages in line. - self.ReceiveMessages(source, messages) - - # We send the client a maximum of self.max_queue_size messages - required_count = max(0, self.max_queue_size - request_comms.queue_size) - - message_list = rdf_flows.MessageList() - # Only give the client messages if we are able to receive them in a - # reasonable time. - if time.time() - now < 10: - client_id = source.Basename() - message_list.job = self.DrainTaskSchedulerQueueForClient( - client_id, required_count) - - # Encode the message_list in the response_comms using the same API version - # the client used. - self._communicator.EncodeMessages( - message_list, - response_comms, - destination=source, - timestamp=timestamp, - api_version=request_comms.api_version) - - return source, len(messages) - - def DrainTaskSchedulerQueueForClient(self, client, max_count=None): - """Drains the client's Task Scheduler queue. - - Args: - client: The client id specifying this client. - max_count: The maximum number of messages we will issue for the client. - If not given, uses self.max_queue_size . - - Returns: - The tasks respresenting the messages returned. If we can not send them, - we can reschedule them for later. - """ - if max_count is None: - max_count = self.max_queue_size - - if max_count <= 0: - return [] - - start_time = time.time() - # Drain the queue for this client - action_requests = data_store.REL_DB.LeaseClientActionRequests( - client, - lease_time=rdfvalue.Duration.From(self.message_expiry_time, - rdfvalue.SECONDS), - limit=max_count) - result = [ - rdf_flow_objects.GRRMessageFromClientActionRequest(r) - for r in action_requests - ] - - GRR_MESSAGES_SENT.Increment(len(result)) - if result: - logging.debug("Drained %d messages for %s in %s seconds.", len(result), - client, - time.time() - start_time) - - return result - - def EnrolFleetspeakClient(self, client_id): + def EnrollFleetspeakClientIfNeeded( + self, client_id: str + ) -> Optional[rdf_objects.ClientMetadata]: """Enrols a Fleetspeak-enabled client for use with GRR. Args: client_id: GRR client-id for the client. Returns: - True if the client is new, and actually got enrolled. This method - is a no-op if the client already exists (in which case False is returned). + None if the client is new, and actually got enrolled. This method + is a no-op if the client already exists (in which case the existing + client metadata is returned). """ client_urn = rdf_client.ClientURN(client_id) # If already enrolled, return. try: - data_store.REL_DB.ReadClientMetadata(client_id) - return False + return data_store.REL_DB.ReadClientMetadata(client_id) except db.UnknownClientError: pass @@ -312,12 +87,13 @@ def EnrolFleetspeakClient(self, client_id): now = rdfvalue.RDFDatetime.Now() data_store.REL_DB.WriteClientMetadata( - client_id, first_seen=now, fleetspeak_enabled=True, last_ping=now) + client_id, first_seen=now, last_ping=now + ) # Publish the client enrollment message. events.Events.PublishEvent( "ClientEnrollment", client_urn, username=FRONTEND_USERNAME) - return True + return None legacy_well_known_session_ids = set([ str(rdfvalue.SessionID(flow_name="Foreman", queue=rdfvalue.RDFURN("W"))), diff --git a/grr/server/grr_response_server/frontend_lib_test.py b/grr/server/grr_response_server/frontend_lib_test.py index 20486a39f4..5782171895 100644 --- a/grr/server/grr_response_server/frontend_lib_test.py +++ b/grr/server/grr_response_server/frontend_lib_test.py @@ -1,71 +1,32 @@ #!/usr/bin/env python """Tests for frontend server, client communicator, and the GRRHTTPClient.""" -import array -import datetime -import logging -import pdb -import time from unittest import mock import zlib from absl import app -from absl import flags from absl.testing import absltest -from cryptography import x509 -from cryptography.hazmat.primitives import hashes -from cryptography.hazmat.primitives.asymmetric import rsa -import requests from google.protobuf import wrappers_pb2 -from grr_response_client import comms -from grr_response_client.client_actions import admin -from grr_response_client.client_actions import standard -from grr_response_core import config -from grr_response_core.lib import queues from grr_response_core.lib import rdfvalue -from grr_response_core.lib import utils -from grr_response_core.lib.rdfvalues import client as rdf_client -from grr_response_core.lib.rdfvalues import crypto as rdf_crypto from grr_response_core.lib.rdfvalues import flows as rdf_flows from grr_response_core.lib.rdfvalues import protodict as rdf_protodict -from grr_response_server import communicator from grr_response_server import data_store from grr_response_server import fleetspeak_connector -from grr_response_server import flow -from grr_response_server import flow_base from grr_response_server import frontend_lib -from grr_response_server import maintenance_utils from grr_response_server import sinks from grr_response_server.databases import db as abstract_db from grr_response_server.databases import db_test_utils from grr_response_server.flows.general import administrative -from grr_response_server.flows.general import ca_enroller from grr_response_server.rdfvalues import flow_objects as rdf_flow_objects from grr_response_server.rdfvalues import objects as rdf_objects from grr_response_server.sinks import test_lib as sinks_test_lib -from grr.test_lib import client_action_test_lib -from grr.test_lib import client_test_lib from grr.test_lib import db_test_lib from grr.test_lib import flow_test_lib -from grr.test_lib import stats_test_lib from grr.test_lib import test_lib -from grr.test_lib import worker_mocks from grr_response_proto import rrg_pb2 -class SendingTestFlow(flow_base.FlowBase): - """Tests that sent messages are correctly collected.""" - - def Start(self): - for i in range(10): - self.CallClient( - client_test_lib.Test, - rdf_protodict.DataBlob(string="test%s" % i), - data=str(i), - next_state="Incoming") - - MESSAGE_EXPIRY_TIME = 100 @@ -76,8 +37,6 @@ def ReceiveMessages(client_id, messages): def TestServer(): return frontend_lib.FrontEndServer( - certificate=config.CONFIG["Frontend.certificate"], - private_key=config.CONFIG["PrivateKeys.server_key"], message_expiry_time=MESSAGE_EXPIRY_TIME) @@ -103,7 +62,7 @@ def testReceiveMessages(self): """Tests receiving messages.""" client_id = "C.1234567890123456" flow_id = "12345678" - data_store.REL_DB.WriteClientMetadata(client_id, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id) _, req = self._FlowSetup(client_id, flow_id) session_id = "%s/%s" % (client_id, flow_id) @@ -125,7 +84,7 @@ def testReceiveMessages(self): def testBlobHandlerMessagesAreHandledOnTheFrontend(self): client_id = "C.1234567890123456" - data_store.REL_DB.WriteClientMetadata(client_id, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id) # Check that the worker queue is empty. self.assertEmpty(data_store.REL_DB.ReadMessageHandlerRequests()) @@ -154,7 +113,7 @@ def testBlobHandlerMessagesAreHandledOnTheFrontend(self): def testCrashReport(self): client_id = "C.1234567890123456" flow_id = "12345678" - data_store.REL_DB.WriteClientMetadata(client_id, fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(client_id) self._FlowSetup(client_id, flow_id) # Make sure the event handler is present. @@ -180,46 +139,6 @@ def testCrashReport(self): self.assertTrue(crash_details_rel) self.assertEqual(crash_details_rel.session_id, session_id) - def testDrainTaskSchedulerQueue(self): - client_id = u"C.1234567890123456" - flow_id = flow.RandomFlowId() - data_store.REL_DB.WriteClientMetadata(client_id, fleetspeak_enabled=False) - - rdf_flow = rdf_flow_objects.Flow( - client_id=client_id, - flow_id=flow_id, - create_time=rdfvalue.RDFDatetime.Now()) - data_store.REL_DB.WriteFlowObject(rdf_flow) - - action_requests = [] - for i in range(3): - data_store.REL_DB.WriteFlowRequests([ - rdf_flow_objects.FlowRequest( - client_id=client_id, flow_id=flow_id, request_id=i) - ]) - - action_requests.append( - rdf_flows.ClientActionRequest( - client_id=client_id, - flow_id=flow_id, - request_id=i, - action_identifier="WmiQuery")) - - data_store.REL_DB.WriteClientActionRequests(action_requests) - server = TestServer() - - res = server.DrainTaskSchedulerQueueForClient(client_id) - msgs = [ - rdf_flow_objects.GRRMessageFromClientActionRequest(r) - for r in action_requests - ] - for r in res: - r.task_id = 0 - for m in msgs: - m.task_id = 0 - - self.assertCountEqual(res, msgs) - class FleetspeakFrontendTests(flow_test_lib.FlowTestsBaseclass): @@ -229,859 +148,16 @@ def testFleetspeakEnrolment(self): # An Enrolment flow should start inline and attempt to send at least # message through fleetspeak as part of the resulting interrogate flow. with mock.patch.object(fleetspeak_connector, "CONN") as mock_conn: - server.EnrolFleetspeakClient(client_id) + server.EnrollFleetspeakClientIfNeeded(client_id) mock_conn.outgoing.InsertMessage.assert_called() -def MakeHTTPException(code=500, msg="Error"): - """A helper for creating a HTTPError exception.""" - response = requests.Response() - response.status_code = code - return requests.ConnectionError(msg, response=response) - - -def MakeResponse(code=500, data=""): - """A helper for creating a HTTPError exception.""" - response = requests.Response() - response.status_code = code - response._content = data - return response - - -class ClientCommsTest(stats_test_lib.StatsTestMixin, - client_action_test_lib.WithAllClientActionsMixin, - test_lib.GRRBaseTest): - """Test the communicator.""" - - def setUp(self): - """Set up communicator tests.""" - super().setUp() - - # These tests change the config so we preserve state. - config_stubber = test_lib.PreserveConfig() - config_stubber.Start() - self.addCleanup(config_stubber.Stop) - - self.client_private_key = config.CONFIG["Client.private_key"] - - self.server_certificate = config.CONFIG["Frontend.certificate"] - self.server_private_key = config.CONFIG["PrivateKeys.server_key"] - self.client_communicator = comms.ClientCommunicator( - private_key=self.client_private_key) - - self.client_communicator.LoadServerCertificate( - server_certificate=self.server_certificate, - ca_certificate=config.CONFIG["CA.certificate"]) - - self.last_urlmock_error = None - - self._SetupCommunicator() - - def _SetupCommunicator(self): - self.server_communicator = frontend_lib.ServerCommunicator( - certificate=self.server_certificate, - private_key=self.server_private_key) - - def ClientServerCommunicate(self, timestamp=None): - """Tests the end to end encrypted communicators.""" - message_list = rdf_flows.MessageList() - for i in range(1, 11): - message_list.job.Append( - session_id=rdfvalue.SessionID( - base="aff4:/flows", queue=queues.FLOWS, flow_name=i), - name="OMG it's a string") - - result = rdf_flows.ClientCommunication() - timestamp = self.client_communicator.EncodeMessages( - message_list, result, timestamp=timestamp) - self.cipher_text = result.SerializeToBytes() - - (decoded_messages, source, client_timestamp) = ( - self.server_communicator.DecryptMessage(self.cipher_text)) - - self.assertEqual(source, self.client_communicator.common_name) - self.assertEqual(client_timestamp, timestamp) - self.assertLen(decoded_messages, 10) - for i in range(1, 11): - self.assertEqual( - decoded_messages[i - 1].session_id, - rdfvalue.SessionID( - base="aff4:/flows", queue=queues.FLOWS, flow_name=i)) - - return decoded_messages - - def testCommunications(self): - """Test that messages from unknown clients are tagged unauthenticated.""" - decoded_messages = self.ClientServerCommunicate() - for i in range(len(decoded_messages)): - self.assertEqual(decoded_messages[i].auth_state, - rdf_flows.GrrMessage.AuthorizationState.UNAUTHENTICATED) - - def _MakeClientRecord(self): - """Make a client in the data store.""" - client_cert = self.ClientCertFromPrivateKey(self.client_private_key) - self.client_id = client_cert.GetCN()[len("aff4:/"):] - data_store.REL_DB.WriteClientMetadata( - self.client_id, fleetspeak_enabled=False, certificate=client_cert) - - def testKnownClient(self): - """Test that messages from known clients are authenticated.""" - self._MakeClientRecord() - - # Now the server should know about it - decoded_messages = self.ClientServerCommunicate() - - for i in range(len(decoded_messages)): - self.assertEqual(decoded_messages[i].auth_state, - rdf_flows.GrrMessage.AuthorizationState.AUTHENTICATED) - - def testClientPingAndClockIsUpdated(self): - """Check PING and CLOCK are updated.""" - self._MakeClientRecord() - - now = rdfvalue.RDFDatetime.Now() - client_now = now - 20 - with test_lib.FakeTime(now): - self.ClientServerCommunicate(timestamp=client_now) - - metadata = data_store.REL_DB.ReadClientMetadata(self.client_id) - - self.assertEqual(now, metadata.ping) - self.assertEqual(client_now, metadata.clock) - - now += 60 - client_now += 40 - with test_lib.FakeTime(now): - self.ClientServerCommunicate(timestamp=client_now) - - metadata = data_store.REL_DB.ReadClientMetadata(self.client_id) - self.assertEqual(now, metadata.ping) - self.assertEqual(client_now, metadata.clock) - - def testClientPingStatsUpdated(self): - """Check client ping stats are updated.""" - self._MakeClientRecord() - - data_store.REL_DB.WriteGRRUser("Test") - - with self.assertStatsCounterDelta( - 1, frontend_lib.CLIENT_PINGS_BY_LABEL, fields=["testlabel"]): - - data_store.REL_DB.AddClientLabels(self.client_id, "Test", ["testlabel"]) - - now = rdfvalue.RDFDatetime.Now() - with test_lib.FakeTime(now): - self.ClientServerCommunicate(timestamp=now) - - def testServerReplayAttack(self): - """Test that replaying encrypted messages to the server invalidates them.""" - self._MakeClientRecord() - - # First send some messages to the server - decoded_messages = self.ClientServerCommunicate(timestamp=1000000) - - encrypted_messages = self.cipher_text - - self.assertEqual(decoded_messages[0].auth_state, - rdf_flows.GrrMessage.AuthorizationState.AUTHENTICATED) - - # Immediate replay is accepted by the server since some proxies do this. - (decoded_messages, _, - _) = self.server_communicator.DecryptMessage(encrypted_messages) - - self.assertEqual(decoded_messages[0].auth_state, - rdf_flows.GrrMessage.AuthorizationState.AUTHENTICATED) - - # Move the client time more than 1h forward. - self.ClientServerCommunicate(timestamp=1000000 + 3700 * 1000000) - - # And replay the old messages again. - (decoded_messages, _, - _) = self.server_communicator.DecryptMessage(encrypted_messages) - - # Messages should now be tagged as desynced - self.assertEqual(decoded_messages[0].auth_state, - rdf_flows.GrrMessage.AuthorizationState.DESYNCHRONIZED) - - def testX509Verify(self): - """X509 Verify can have several failure paths.""" - - # This is a successful verify. - with mock.patch.object( - rdf_crypto.RDFX509Cert, "Verify", lambda self, public_key=None: True): - self.client_communicator.LoadServerCertificate( - self.server_certificate, config.CONFIG["CA.certificate"]) - - def Verify(_, public_key=False): - _ = public_key - raise rdf_crypto.VerificationError("Testing verification failure.") - - # Mock the verify function to simulate certificate failures. - with mock.patch.object(rdf_crypto.RDFX509Cert, "Verify", Verify): - self.assertRaises(IOError, self.client_communicator.LoadServerCertificate, - self.server_certificate, - config.CONFIG["CA.certificate"]) - - def testErrorDetection(self): - """Tests the end to end encrypted communicators.""" - # Install the client - now we can verify its signed messages - self._MakeClientRecord() - - # Make something to send - message_list = rdf_flows.MessageList() - for i in range(0, 10): - message_list.job.Append(session_id=str(i)) - - result = rdf_flows.ClientCommunication() - self.client_communicator.EncodeMessages(message_list, result) - - cipher_text = result.SerializeToBytes() - - # Depending on this modification several things may happen: - # 1) The padding may not match which will cause a decryption exception. - # 2) The protobuf may fail to decode causing a decoding exception. - # 3) The modification may affect the signature resulting in UNAUTHENTICATED - # messages. - # 4) The modification may have no effect on the data at all. - for x in range(0, len(cipher_text), 50): - # Futz with the cipher text (Make sure it's really changed) - mod = chr((cipher_text[x] % 250) + 1).encode("latin-1") - mod_cipher_text = cipher_text[:x] + mod + cipher_text[x + 1:] - - try: - decoded, client_id, _ = self.server_communicator.DecryptMessage( - mod_cipher_text) - - for i, message in enumerate(decoded): - # If the message is actually authenticated it must not be changed! - if message.auth_state == message.AuthorizationState.AUTHENTICATED: - self.assertEqual(message.source, client_id) - - # These fields are set by the decoder and are not present in the - # original message - so we clear them before comparison. - message.auth_state = None - message.source = None - self.assertEqual(message, message_list.job[i]) - else: - logging.debug("Message %s: Authstate: %s", i, message.auth_state) - - except communicator.DecodingError as e: - logging.debug("Detected alteration at %s: %s", x, e) - - def testEnrollingCommunicator(self): - """Test that the ClientCommunicator generates good keys.""" - self.client_communicator = comms.ClientCommunicator() - - self.client_communicator.LoadServerCertificate( - self.server_certificate, config.CONFIG["CA.certificate"]) - - # Verify that the CN is of the correct form - csr = self.client_communicator.GetCSR() - cn = rdf_client.ClientURN.FromPublicKey(csr.GetPublicKey()) - self.assertEqual(cn, csr.GetCN()) - - def testServerKeyRotation(self): - self._MakeClientRecord() - - # Now the server should know about the client. - decoded_messages = self.ClientServerCommunicate() - for i in range(len(decoded_messages)): - self.assertEqual(decoded_messages[i].auth_state, - rdf_flows.GrrMessage.AuthorizationState.AUTHENTICATED) - - # Suppress the output. - with mock.patch.object(maintenance_utils, "EPrint", lambda msg: None): - maintenance_utils.RotateServerKey() - - server_certificate = config.CONFIG["Frontend.certificate"] - server_private_key = config.CONFIG["PrivateKeys.server_key"] - - self.assertNotEqual(server_certificate, self.server_certificate) - self.assertNotEqual(server_private_key, self.server_private_key) - - self.server_communicator = frontend_lib.ServerCommunicator( - certificate=server_certificate, private_key=server_private_key) - - # Clients can't connect at this point since they use the outdated - # session key. - with self.assertRaises(communicator.DecryptionError): - self.ClientServerCommunicate() - - # After the client reloads the server cert, this should start - # working again. - self.client_communicator.LoadServerCertificate( - server_certificate=server_certificate, - ca_certificate=config.CONFIG["CA.certificate"]) - - self.assertLen(list(self.ClientServerCommunicate()), 10) - - -class HTTPClientTests(client_action_test_lib.WithAllClientActionsMixin, - test_lib.GRRBaseTest): - """Test the http communicator.""" - - def setUp(self): - """Set up communicator tests.""" - super().setUp() - - # These tests change the config so we preserve state. - config_stubber = test_lib.PreserveConfig() - config_stubber.Start() - self.addCleanup(config_stubber.Stop) - - self.server_private_key = config.CONFIG["PrivateKeys.server_key"] - self.server_certificate = config.CONFIG["Frontend.certificate"] - - # Make a new client - self.CreateNewClientObject() - - # And cache it in the server - self.CreateNewServerCommunicator() - - requests_stubber = mock.patch.object(requests, "request", self.UrlMock) - requests_stubber.start() - self.addCleanup(requests_stubber.stop) - - sleep_stubber = mock.patch.object(time, "sleep", lambda x: None) - sleep_stubber.start() - self.addCleanup(sleep_stubber.stop) - - self.messages = [] - - ca_enroller.enrolment_cache.Flush() - - # Response to send back to clients. - self.server_response = dict( - session_id="aff4:/W:session", name="Echo", response_id=2) - - def _MakeClient(self): - self.client_certificate = self.ClientCertFromPrivateKey( - config.CONFIG["Client.private_key"]) - self.client_cn = self.client_certificate.GetCN() - self.client_id = self.client_cn[len("aff4:/"):] - - data_store.REL_DB.WriteClientMetadata( - self.client_id, - certificate=self.client_certificate, - fleetspeak_enabled=False) - - def _ClearClient(self): - del data_store.REL_DB.delegate.metadatas[self.client_id] - - def CreateNewServerCommunicator(self): - self._MakeClient() - self.server_communicator = frontend_lib.ServerCommunicator( - certificate=self.server_certificate, - private_key=self.server_private_key) - - def CreateClientCommunicator(self): - self.client_communicator = comms.GRRHTTPClient( - ca_cert=config.CONFIG["CA.certificate"], - worker_cls=worker_mocks.ClientWorker) - - def CreateNewClientObject(self): - self.CreateClientCommunicator() - - # Disable stats collection for tests. - self.client_communicator.client_worker.last_stats_sent_time = ( - time.time() + 3600) - - # Build a client context with preloaded server certificates - self.client_communicator.communicator.LoadServerCertificate( - self.server_certificate, config.CONFIG["CA.certificate"]) - - self.client_communicator.http_manager.retry_error_limit = 5 - - def UrlMock(self, num_messages=10, url=None, data=None, **kwargs): - """A mock for url handler processing from the server's POV.""" - if "server.pem" in url: - cert = str(config.CONFIG["Frontend.certificate"]).encode("ascii") - return MakeResponse(200, cert) - - _ = kwargs - try: - comms_cls = rdf_flows.ClientCommunication - self.client_communication = comms_cls.FromSerializedBytes(data) - - # Decrypt incoming messages - self.messages, source, ts = self.server_communicator.DecodeMessages( - self.client_communication) - - # Make sure the messages are correct - self.assertEqual(source, self.client_cn) - messages = sorted( - [m for m in self.messages if m.session_id == "aff4:/W:session"], - key=lambda m: m.response_id) - self.assertEqual([m.response_id for m in messages], - list(range(len(messages)))) - self.assertEqual([m.request_id for m in messages], [1] * len(messages)) - - # Now prepare a response - response_comms = rdf_flows.ClientCommunication() - message_list = rdf_flows.MessageList() - for i in range(0, num_messages): - message_list.job.Append(request_id=i, **self.server_response) - - # Preserve the timestamp as a nonce - self.server_communicator.EncodeMessages( - message_list, - response_comms, - destination=source, - timestamp=ts, - api_version=self.client_communication.api_version) - - return MakeResponse(200, response_comms.SerializeToBytes()) - except communicator.UnknownClientCertError: - raise MakeHTTPException(406) - except Exception as e: - logging.info("Exception in mock urllib.request.Open: %s.", e) - self.last_urlmock_error = e - - if flags.FLAGS.pdb_post_mortem: - pdb.post_mortem() - - raise MakeHTTPException(500) - - def CheckClientQueue(self): - """Checks that the client context received all server messages.""" - # Check the incoming messages - self.assertEqual(self.client_communicator.client_worker.InQueueSize(), 10) - - for i, message in enumerate( - self.client_communicator.client_worker._in_queue.queue): - # This is the common name embedded in the certificate. - self.assertEqual(message.source, "aff4:/GRR Test Server") - self.assertEqual(message.response_id, 2) - self.assertEqual(message.request_id, i) - self.assertEqual(message.session_id, "aff4:/W:session") - self.assertEqual(message.auth_state, - rdf_flows.GrrMessage.AuthorizationState.AUTHENTICATED) - - # Clear the queue - self.client_communicator.client_worker._in_queue.queue.clear() - - def SendToServer(self): - """Schedule some packets from client to server.""" - # Generate some client traffic - for i in range(0, 10): - self.client_communicator.client_worker.SendReply( - rdf_flows.GrrStatus(), - session_id=rdfvalue.SessionID("W:session"), - response_id=i, - request_id=1) - - def testInitialEnrollment(self): - """If the client has no certificate initially it should enroll.""" - - # Clear the certificate so we can generate a new one. - with test_lib.ConfigOverrider({ - "Client.private_key": "", - }): - self.CreateNewClientObject() - - # Client should get a new Common Name. - self.assertNotEqual(self.client_cn, - self.client_communicator.communicator.common_name) - - self.client_cn = self.client_communicator.communicator.common_name - - # The client will sleep and re-attempt to connect multiple times. - status = self.client_communicator.RunOnce() - - self.assertEqual(status.code, 406) - - # The client should now send an enrollment request. - self.client_communicator.RunOnce() - - # Client should generate enrollment message by itself. - self.assertLen(self.messages, 1) - self.assertEqual(self.messages[0].session_id.Basename(), - "E:%s" % ca_enroller.EnrolmentHandler.handler_name) - - def testEnrollment(self): - """Test the http response to unknown clients.""" - - self._ClearClient() - - # Now communicate with the server. - self.SendToServer() - status = self.client_communicator.RunOnce() - - # We expect to receive a 406 and all client messages will be tagged as - # UNAUTHENTICATED. - self.assertEqual(status.code, 406) - self.assertLen(self.messages, 10) - self.assertEqual(self.messages[0].auth_state, - rdf_flows.GrrMessage.AuthorizationState.UNAUTHENTICATED) - - # The next request should be an enrolling request. - self.client_communicator.RunOnce() - - self.assertLen(self.messages, 11) - enrolment_messages = [] - - expected_id = "E:%s" % ca_enroller.EnrolmentHandler.handler_name - for m in self.messages: - if m.session_id.Basename() == expected_id: - enrolment_messages.append(m) - - self.assertLen(enrolment_messages, 1) - - # Now we manually run the enroll well known flow with the enrollment - # request. This will start a new flow for enrolling the client, sign the - # cert and add it to the data store. - handler = ca_enroller.EnrolmentHandler() - req = rdf_objects.MessageHandlerRequest( - client_id=self.client_id, request=enrolment_messages[0].payload) - handler.ProcessMessages([req]) - - # The next client communication should be enrolled now. - status = self.client_communicator.RunOnce() - - self.assertEqual(status.code, 200) - - # There should be a cert for the client right now. - md = data_store.REL_DB.ReadClientMetadata(self.client_id) - self.assertTrue(md.certificate) - - # Now communicate with the server once again. - self.SendToServer() - status = self.client_communicator.RunOnce() - - self.assertEqual(status.code, 200) - - def testEnrollmentHandler(self): - self._ClearClient() - - # First 406 queues an EnrolmentRequest. - status = self.client_communicator.RunOnce() - self.assertEqual(status.code, 406) - - # Send it to the server. - status = self.client_communicator.RunOnce() - self.assertEqual(status.code, 406) - - self.assertLen(self.messages, 1) - self.assertEqual(self.messages[0].session_id.Basename(), - "E:%s" % ca_enroller.EnrolmentHandler.handler_name) - - request = rdf_objects.MessageHandlerRequest( - client_id=self.messages[0].source.Basename(), - handler_name="Enrol", - request_id=12345, - request=self.messages[0].payload) - - handler = ca_enroller.EnrolmentHandler() - handler.ProcessMessages([request]) - - # The next client communication should give a 200. - status = self.client_communicator.RunOnce() - self.assertEqual(status.code, 200) - - def testReboots(self): - """Test the http communication with reboots.""" - # Now we add the new client record to the server cache - self.SendToServer() - self.client_communicator.RunOnce() - self.CheckClientQueue() - - # Simulate the client rebooted - self.CreateNewClientObject() - - self.SendToServer() - self.client_communicator.RunOnce() - self.CheckClientQueue() - - # Simulate the server rebooting - self.CreateNewServerCommunicator() - - self.SendToServer() - self.client_communicator.RunOnce() - self.CheckClientQueue() - - def _CheckFastPoll(self, require_fastpoll, expected_sleeptime): - self.server_response = dict( - session_id="aff4:/W:session", - name="Echo", - response_id=2, - require_fastpoll=require_fastpoll) - - # Make sure we don't have any output messages that might override the - # fastpoll setting from the input messages we send - self.assertEqual(self.client_communicator.client_worker.OutQueueSize(), 0) - - self.client_communicator.RunOnce() - # Make sure the timer is set to the correct value. - self.assertEqual(self.client_communicator.timer.sleep_time, - expected_sleeptime) - self.CheckClientQueue() - - def testNoFastPoll(self): - """Test that the fast poll False is respected on input messages. - - Also make sure we wait the correct amount of time before next poll. - """ - self._CheckFastPoll(False, config.CONFIG["Client.poll_max"]) - - def testFastPoll(self): - """Test that the fast poll True is respected on input messages. - - Also make sure we wait the correct amount of time before next poll. - """ - self._CheckFastPoll(True, config.CONFIG["Client.poll_min"]) - - def testCorruption(self): - """Simulate corruption of the http payload.""" - - self.corruptor_field = None - - def Corruptor(url="", data=None, **kwargs): - """Futz with some of the fields.""" - comm_cls = rdf_flows.ClientCommunication - if data is not None: - self.client_communication = comm_cls.FromSerializedBytes(data) - else: - self.client_communication = comm_cls(None) - - if self.corruptor_field and "server.pem" not in url: - orig_str_repr = self.client_communication.SerializeToBytes() - field_data = getattr(self.client_communication, self.corruptor_field) - if hasattr(field_data, "SerializeToBytes"): - # This converts encryption keys to a string so we can corrupt them. - field_data = field_data.SerializeToBytes() - - modified_data = array.array("B", field_data) - offset = len(field_data) // 2 - char = field_data[offset] - modified_data[offset] = char % 250 + 1 - setattr(self.client_communication, self.corruptor_field, - modified_data.tobytes()) - - # Make sure we actually changed the data. - self.assertNotEqual(field_data, modified_data) - - mod_str_repr = self.client_communication.SerializeToBytes() - self.assertLen(orig_str_repr, len(mod_str_repr)) - differences = [ - True for x, y in zip(orig_str_repr, mod_str_repr) if x != y - ] - self.assertLen(differences, 1) - - data = self.client_communication.SerializeToBytes() - return self.UrlMock(url=url, data=data, **kwargs) - - with mock.patch.object(requests, "request", Corruptor): - self.SendToServer() - status = self.client_communicator.RunOnce() - self.assertEqual(status.code, 200) - - for field in ["packet_iv", "encrypted"]: - # Corrupting each field should result in HMAC verification errors. - self.corruptor_field = field - - self.SendToServer() - status = self.client_communicator.RunOnce() - - self.assertEqual(status.code, 500) - self.assertIn("HMAC verification failed", str(self.last_urlmock_error)) - - # Corruption of these fields will likely result in RSA errors, since we do - # the RSA operations before the HMAC verification (in order to recover the - # hmac key): - for field in ["encrypted_cipher", "encrypted_cipher_metadata"]: - # Corrupting each field should result in HMAC verification errors. - self.corruptor_field = field - - self.SendToServer() - status = self.client_communicator.RunOnce() - - self.assertEqual(status.code, 500) - - def testClientRetransmission(self): - """Test that client retransmits failed messages.""" - fail = True - num_messages = 10 - - def FlakyServer(url=None, **kwargs): - if not fail or "server.pem" in url: - return self.UrlMock(num_messages=num_messages, url=url, **kwargs) - - raise MakeHTTPException(500) - - with mock.patch.object(requests, "request", FlakyServer): - self.SendToServer() - status = self.client_communicator.RunOnce() - self.assertEqual(status.code, 500) - - # Server should not receive anything. - self.assertEmpty(self.messages) - - # Try to send these messages again. - fail = False - - self.assertEqual(self.client_communicator.client_worker.InQueueSize(), 0) - - status = self.client_communicator.RunOnce() - - self.assertEqual(status.code, 200) - - # We have received 10 client messages. - self.assertEqual(self.client_communicator.client_worker.InQueueSize(), 10) - self.CheckClientQueue() - - # Server should have received 10 messages this time. - self.assertLen(self.messages, 10) - - # TODO(hanuszczak): We have a separate test suite for the stat collector. - # Most of these test methods are no longer required, especially that now they - # need to use implementation-specific methods instead of the public API. - - def testClientStatsCollection(self): - """Tests that the client stats are collected automatically.""" - now = 1000000 - # Pretend we have already sent stats. - self.client_communicator.client_worker.stats_collector._last_send_time = ( - rdfvalue.RDFDatetime.FromSecondsSinceEpoch(now)) - - with test_lib.FakeTime(now): - self.client_communicator.client_worker.stats_collector._Send() - - runs = [] - with mock.patch.object(admin.GetClientStatsAuto, "Run", - lambda cls, _: runs.append(1)): - - # No stats collection after 10 minutes. - with test_lib.FakeTime(now + 600): - self.client_communicator.client_worker.stats_collector._Send() - self.assertEmpty(runs) - - # Let one hour pass. - with test_lib.FakeTime(now + 3600): - self.client_communicator.client_worker.stats_collector._Send() - # This time the client should collect stats. - self.assertLen(runs, 1) - - # Let one hour and ten minutes pass. - with test_lib.FakeTime(now + 3600 + 600): - self.client_communicator.client_worker.stats_collector._Send() - # Again, there should be no stats collection, as last collection - # happened less than an hour ago. - self.assertLen(runs, 1) - - def testClientStatsCollectionHappensEveryMinuteWhenClientIsBusy(self): - """Tests that client stats are collected more often when client is busy.""" - now = 1000000 - # Pretend we have already sent stats. - self.client_communicator.client_worker.stats_collector._last_send_time = ( - rdfvalue.RDFDatetime.FromSecondsSinceEpoch(now)) - self.client_communicator.client_worker._is_active = True - - with test_lib.FakeTime(now): - self.client_communicator.client_worker.stats_collector._Send() - - runs = [] - with mock.patch.object(admin.GetClientStatsAuto, "Run", - lambda cls, _: runs.append(1)): - - # No stats collection after 30 seconds. - with test_lib.FakeTime(now + 30): - self.client_communicator.client_worker.stats_collector._Send() - self.assertEmpty(runs) - - # Let 61 seconds pass. - with test_lib.FakeTime(now + 61): - self.client_communicator.client_worker.stats_collector._Send() - # This time the client should collect stats. - self.assertLen(runs, 1) - - # No stats collection within one minute from the last time. - with test_lib.FakeTime(now + 61 + 59): - self.client_communicator.client_worker.stats_collector._Send() - self.assertLen(runs, 1) - - # Stats collection happens as more than one minute has passed since the - # last one. - with test_lib.FakeTime(now + 61 + 61): - self.client_communicator.client_worker.stats_collector._Send() - self.assertLen(runs, 2) - - def testClientStatsCollectionAlwaysHappensAfterHandleMessage(self): - """Tests that client stats are collected more often when client is busy.""" - now = 1000000 - # Pretend we have already sent stats. - self.client_communicator.client_worker.stats_collector._last_send_time = ( - rdfvalue.RDFDatetime.FromSecondsSinceEpoch(now)) - - with test_lib.FakeTime(now): - self.client_communicator.client_worker.stats_collector._Send() - - runs = [] - with mock.patch.object(admin.GetClientStatsAuto, "Run", - lambda cls, _: runs.append(1)): - - # No stats collection after 30 seconds. - with test_lib.FakeTime(now + 30): - self.client_communicator.client_worker.stats_collector._Send() - self.assertEmpty(runs) - - msg = rdf_flows.GrrMessage( - name=standard.HashFile.__name__, generate_task_id=True) - self.client_communicator.client_worker.HandleMessage(msg) - - # HandleMessage was called, but one minute hasn't passed, so - # stats should not be sent. - with test_lib.FakeTime(now + 59): - self.client_communicator.client_worker.stats_collector._Send() - self.assertEmpty(runs) - - # HandleMessage was called more than one minute ago, so stats - # should be sent. - with test_lib.FakeTime(now + 61): - self.client_communicator.client_worker.stats_collector._Send() - self.assertLen(runs, 1) - - def RaiseError(self, **_): - raise MakeHTTPException(500, "Not a real connection.") - - def testClientConnectionErrors(self): - client_obj = comms.GRRHTTPClient(worker_cls=worker_mocks.ClientWorker) - # Make the connection unavailable and skip the retry interval. - with utils.MultiStubber( - (requests, "request", self.RaiseError), - (client_obj.http_manager, "connection_error_limit", 8)): - # Simulate a client run. The client will retry the connection limit by - # itself. The Run() method will quit when connection_error_limit is - # reached. This will make the real client quit. - client_obj.Run() - - self.assertEqual(client_obj.http_manager.consecutive_connection_errors, 9) - - class FrontEndServerTest(absltest.TestCase): def setUp(self): super().setUp() - private_key = rsa.generate_private_key(65537, 2048) - certificate = x509.CertificateBuilder( - subject_name=x509.Name([ - x509.NameAttribute(x509.NameOID.COMMON_NAME, "GRR Frontend"), - ]), - issuer_name=x509.Name([ - x509.NameAttribute(x509.NameOID.COMMON_NAME, "GRR"), - ]), - serial_number=x509.random_serial_number(), - not_valid_before=datetime.datetime.now() - datetime.timedelta(days=1), - not_valid_after=datetime.datetime.now() + datetime.timedelta(days=1), - public_key=private_key.public_key(), - ).sign(private_key, hashes.SHA256()) - - rdf_private_key = rdf_crypto.RSAPrivateKey(private_key) - rdf_certificate = rdf_crypto.RDFX509Cert(certificate) - self.server = frontend_lib.FrontEndServer( - private_key=rdf_private_key, - certificate=rdf_certificate, ) @db_test_lib.WithDatabase diff --git a/grr/server/grr_response_server/gui/api_labels_restricted_call_router_test.py b/grr/server/grr_response_server/gui/api_labels_restricted_call_router_test.py index 6b804c3e79..61f318a028 100644 --- a/grr/server/grr_response_server/gui/api_labels_restricted_call_router_test.py +++ b/grr/server/grr_response_server/gui/api_labels_restricted_call_router_test.py @@ -263,8 +263,9 @@ def CheckOnlyFollowingMethodsArePermitted(self, router, method_names): self.assertTrue(status, "%s must be permitted, but it's not" % method_name) else: - self.assertFalse(status, - "%s mut not be permitted, but it is" % method_name) + self.assertFalse( + status, "%s must not be permitted, but it is" % method_name + ) def testReturnsCustomHandlerForSearchClients(self): router = api_router.ApiLabelsRestrictedCallRouter() diff --git a/grr/server/grr_response_server/gui/api_plugins/client.py b/grr/server/grr_response_server/gui/api_plugins/client.py index 73689c6748..6932a95146 100644 --- a/grr/server/grr_response_server/gui/api_plugins/client.py +++ b/grr/server/grr_response_server/gui/api_plugins/client.py @@ -44,8 +44,7 @@ def UpdateClientsFromFleetspeak(clients): return id_map = {} for client in clients: - if client.fleetspeak_enabled: - id_map[fleetspeak_utils.GRRIDToFleetspeakID(client.client_id)] = client + id_map[fleetspeak_utils.GRRIDToFleetspeakID(client.client_id)] = client if not id_map: return res = fleetspeak_connector.CONN.outgoing.ListClients( @@ -210,7 +209,6 @@ def InitFromClientInfo( self.last_clock = md.clock if md.last_crash_timestamp: self.last_crash_at = md.last_crash_timestamp - self.fleetspeak_enabled = md.fleetspeak_enabled self.labels = client_info.labels @@ -604,17 +602,7 @@ class ApiGetLastClientIPAddressHandler(api_call_handler_base.ApiCallHandler): def Handle(self, args, context=None): client_id = str(args.client_id) - md = data_store.REL_DB.ReadClientMetadata(client_id) - if md.fleetspeak_enabled: - ip_str, ipaddr_obj = _GetAddrFromFleetspeak(client_id) - else: - try: - ipaddr_obj = md.ip.AsIPAddr() - ip_str = str(ipaddr_obj) - except ValueError: - ipaddr_obj = None - ip_str = "" - + ip_str, ipaddr_obj = _GetAddrFromFleetspeak(client_id) status, info = ip_resolver.IP_RESOLVER.RetrieveIPInfo(ipaddr_obj) return ApiGetLastClientIPAddressResult(ip=ip_str, info=info, status=status) diff --git a/grr/server/grr_response_server/gui/api_plugins/client_test.py b/grr/server/grr_response_server/gui/api_plugins/client_test.py index 7a57e4d53c..656a096a9e 100644 --- a/grr/server/grr_response_server/gui/api_plugins/client_test.py +++ b/grr/server/grr_response_server/gui/api_plugins/client_test.py @@ -146,8 +146,7 @@ def setUp(self): self.handler = client_plugin.ApiRemoveClientsLabelsHandler() def testRemovesUserLabelFromSingleClient(self): - data_store.REL_DB.WriteClientMetadata( - self.client_ids[0], fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(self.client_ids[0]) data_store.REL_DB.AddClientLabels(self.client_ids[0], self.context.username, [u"foo", u"bar"]) @@ -162,8 +161,7 @@ def testRemovesUserLabelFromSingleClient(self): self.assertEqual(labels[0].owner, self.context.username) def testDoesNotRemoveSystemLabelFromSingleClient(self): - data_store.REL_DB.WriteClientMetadata( - self.client_ids[0], fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(self.client_ids[0]) data_store.REL_DB.WriteGRRUser("GRR") data_store.REL_DB.AddClientLabels(self.client_ids[0], u"GRR", [u"foo"]) idx = client_index.ClientIndex() @@ -183,8 +181,7 @@ def testRemovesUserLabelWhenSystemLabelWithSimilarNameAlsoExists(self): data_store.REL_DB.WriteGRRUser(self.context.username) data_store.REL_DB.WriteGRRUser("GRR") - data_store.REL_DB.WriteClientMetadata( - self.client_ids[0], fleetspeak_enabled=False) + data_store.REL_DB.WriteClientMetadata(self.client_ids[0]) data_store.REL_DB.AddClientLabels(self.client_ids[0], self.context.username, [u"foo"]) data_store.REL_DB.AddClientLabels(self.client_ids[0], u"GRR", [u"foo"]) @@ -356,6 +353,68 @@ def _SetUpClient(self): self.client_id = self.SetupClient(0) +class ApiGetClientVersionsTest(api_test_lib.ApiCallHandlerTest): + """Test for ApiGetClientVersions using the relational db.""" + + def setUp(self): + super().setUp() + self.handler = client_plugin.ApiGetClientVersionsHandler() + + def testReturnsAll(self): + self.client_id = self.SetupClient(0) + self.fqdn = "test1234.examples.com" + kernels = [42, 45, 100] + + for time in kernels: + with test_lib.FakeTime(time): + client = rdf_objects.ClientSnapshot( + client_id=self.client_id, kernel=f"{time}" + ) + client.knowledge_base.fqdn = self.fqdn + data_store.REL_DB.WriteClientSnapshot(client) + + args = client_plugin.ApiGetClientVersionsArgs(client_id=self.client_id) + with test_lib.FakeTime(101): + result = self.handler.Handle(args, context=self.context) + + self.assertLen(result.items, 3) + self.assertEqual(result.items[0].client_id, self.client_id) + self.assertEqual(result.items[0].knowledge_base.fqdn, self.fqdn) + self.assertEqual(result.items[0].os_info.kernel, f"{kernels[0]}") + self.assertEqual(result.items[1].client_id, self.client_id) + self.assertEqual(result.items[1].knowledge_base.fqdn, self.fqdn) + self.assertEqual(result.items[1].os_info.kernel, f"{kernels[1]}") + self.assertEqual(result.items[2].client_id, self.client_id) + self.assertEqual(result.items[2].knowledge_base.fqdn, self.fqdn) + self.assertEqual(result.items[2].os_info.kernel, f"{kernels[2]}") + + def testFiltersStartAndEnd(self): + self.client_id = self.SetupClient(0) + self.fqdn = "test1234.examples.com" + kernels = [42, 45, 100] + + for time in kernels: + with test_lib.FakeTime(time): + client = rdf_objects.ClientSnapshot( + client_id=self.client_id, kernel=f"{time}" + ) + client.knowledge_base.fqdn = self.fqdn + data_store.REL_DB.WriteClientSnapshot(client) + + args = client_plugin.ApiGetClientVersionsArgs( + client_id=self.client_id, + start=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(43), + end=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(53), + ) + with test_lib.FakeTime(101): + result = self.handler.Handle(args, context=self.context) + + self.assertLen(result.items, 1) + self.assertEqual(result.items[0].client_id, self.client_id) + self.assertEqual(result.items[0].knowledge_base.fqdn, self.fqdn) + self.assertEqual(result.items[0].os_info.kernel, f"{kernels[1]}") + + def TSProtoFromString(string): ts = timestamp_pb2.Timestamp() ts.FromJsonString(string) @@ -369,10 +428,9 @@ def testUpdateClientsFromFleetspeak(self): client_id_2 = client_plugin.ApiClientId("C." + "2" * 16) client_id_3 = client_plugin.ApiClientId("C." + "3" * 16) clients = [ - client_plugin.ApiClient(client_id=client_id_1, fleetspeak_enabled=True), - client_plugin.ApiClient(client_id=client_id_2, fleetspeak_enabled=True), - client_plugin.ApiClient( - client_id=client_id_3, fleetspeak_enabled=False), + client_plugin.ApiClient(client_id=client_id_1), + client_plugin.ApiClient(client_id=client_id_2), + client_plugin.ApiClient(client_id=client_id_3), ] conn = mock.MagicMock() conn.outgoing.ListClients.return_value = admin_pb2.ListClientsResponse( @@ -388,24 +446,30 @@ def testUpdateClientsFromFleetspeak(self): ]) with mock.patch.object(fleetspeak_connector, "CONN", conn): client_plugin.UpdateClientsFromFleetspeak(clients) - self.assertEqual(clients, [ - client_plugin.ApiClient( - client_id=client_id_1, - fleetspeak_enabled=True, - last_seen_at=rdfvalue.RDFDatetime.FromHumanReadable( - "2018-01-01T00:00:01Z"), - last_clock=rdfvalue.RDFDatetime.FromHumanReadable( - "2018-01-01T00:00:02Z")), - client_plugin.ApiClient( - client_id=client_id_2, - fleetspeak_enabled=True, - last_seen_at=rdfvalue.RDFDatetime.FromHumanReadable( - "2018-01-02T00:00:01Z"), - last_clock=rdfvalue.RDFDatetime.FromHumanReadable( - "2018-01-02T00:00:02Z")), - client_plugin.ApiClient( - client_id=client_id_3, fleetspeak_enabled=False), - ]) + self.assertEqual( + clients, + [ + client_plugin.ApiClient( + client_id=client_id_1, + last_seen_at=rdfvalue.RDFDatetime.FromHumanReadable( + "2018-01-01T00:00:01Z" + ), + last_clock=rdfvalue.RDFDatetime.FromHumanReadable( + "2018-01-01T00:00:02Z" + ), + ), + client_plugin.ApiClient( + client_id=client_id_2, + last_seen_at=rdfvalue.RDFDatetime.FromHumanReadable( + "2018-01-02T00:00:01Z" + ), + last_clock=rdfvalue.RDFDatetime.FromHumanReadable( + "2018-01-02T00:00:02Z" + ), + ), + client_plugin.ApiClient(client_id=client_id_3), + ], + ) def testGetAddrFromFleetspeakIpV4(self): client_id = client_plugin.ApiClientId("C." + "1" * 16) diff --git a/grr/server/grr_response_server/gui/api_plugins/flow.py b/grr/server/grr_response_server/gui/api_plugins/flow.py index 903a0dd4c3..5defce8b96 100644 --- a/grr/server/grr_response_server/gui/api_plugins/flow.py +++ b/grr/server/grr_response_server/gui/api_plugins/flow.py @@ -290,7 +290,7 @@ def InitFromFlowObject(self, self.runner_args.cpu_limit = flow_obj.cpu_limit if flow_obj.HasField("network_bytes_limit"): - self.runner_args.cpu_limit = flow_obj.network_bytes_limit + self.runner_args.network_bytes_limit = flow_obj.network_bytes_limit if flow_obj.original_flow.flow_id: self.original_flow = ApiFlowReference().FromFlowReference( diff --git a/grr/server/grr_response_server/gui/api_plugins/flow_regression_test.py b/grr/server/grr_response_server/gui/api_plugins/flow_regression_test.py index c81d5ae003..a9ca8ab105 100644 --- a/grr/server/grr_response_server/gui/api_plugins/flow_regression_test.py +++ b/grr/server/grr_response_server/gui/api_plugins/flow_regression_test.py @@ -43,7 +43,10 @@ def Run(self): flow_id = flow_test_lib.StartFlow( discovery.Interrogate, client_id=client_id, - creator=self.test_username) + network_bytes_limit=8192, + cpu_limit=60, + creator=self.test_username, + ) replace = api_regression_test_lib.GetFlowTestReplaceDict( client_id, flow_id, "F:ABCDEF12") diff --git a/grr/server/grr_response_server/gui/api_plugins/flow_test.py b/grr/server/grr_response_server/gui/api_plugins/flow_test.py index 357cd0ac9c..5bfc03749e 100644 --- a/grr/server/grr_response_server/gui/api_plugins/flow_test.py +++ b/grr/server/grr_response_server/gui/api_plugins/flow_test.py @@ -48,6 +48,7 @@ from grr_response_server.rdfvalues import objects as rdf_objects from grr.test_lib import action_mocks from grr.test_lib import db_test_lib +from grr.test_lib import fleetspeak_test_lib from grr.test_lib import flow_test_lib from grr.test_lib import hunt_test_lib from grr.test_lib import parser_test_lib @@ -119,28 +120,6 @@ def testWithFlowProgressTypeReportsProgressCorrectly(self): self.assertEqual(flow_api_obj.progress.status, "Progress.") -class ApiCreateFlowHandlerTest(api_test_lib.ApiCallHandlerTest): - """Test for ApiCreateFlowHandler.""" - - def setUp(self): - super().setUp() - self.client_id = self.SetupClient(0) - self.handler = flow_plugin.ApiCreateFlowHandler() - - def testRunnerArgsBaseSessionIdDoesNotAffectCreatedFlow(self): - """When multiple clients match, check we run on the latest one.""" - flow_runner_args = rdf_flow_runner.FlowRunnerArgs( - base_session_id="aff4:/foo") - args = flow_plugin.ApiCreateFlowArgs( - client_id=self.client_id, - flow=flow_plugin.ApiFlow( - name=processes.ListProcesses.__name__, - runner_args=flow_runner_args)) - - result = self.handler.Handle(args, context=self.context) - self.assertNotStartsWith(str(result.urn), "aff4:/foo") - - class ApiGetFlowFilesArchiveHandlerTest(api_test_lib.ApiCallHandlerTest): """Tests for ApiGetFlowFilesArchiveHandler.""" @@ -235,34 +214,6 @@ def testGeneratesZipArchiveForCollectFilesByKnownPath(self): self.assertEqual(manifest["processed_files"], 1) self.assertEqual(manifest["ignored_files"], 0) - def testGeneratesZipArchiveForCollectSingleFile(self): - client_mock = action_mocks.FileFinderClientMockWithTimestamps() - with temp.AutoTempDirPath(remove_non_empty=True) as temp_dir: - temp_filepath = os.path.join(temp_dir, "foo.txt") - with open(temp_filepath, mode="w") as temp_file: - temp_file.write("Lorem ipsum.") - - flow_id = flow_test_lib.TestFlowHelper( - file.CollectSingleFile.__name__, - client_mock, - client_id=self.client_id, - path=temp_filepath, - creator=self.test_username, - ) - - result = self.handler.Handle( - flow_plugin.ApiGetFlowFilesArchiveArgs( - client_id=self.client_id, flow_id=flow_id, archive_format="ZIP" - ), - context=self.context, - ) - manifest = self._GetZipManifest(result) - - self.assertEqual(manifest["archived_files"], 1) - self.assertEqual(manifest["failed_files"], 0) - self.assertEqual(manifest["processed_files"], 1) - self.assertEqual(manifest["ignored_files"], 0) - def testGeneratesZipArchiveForCollectMultipleFiles(self): client_mock = action_mocks.CollectMultipleFilesClientMock() with temp.AutoTempDirPath(remove_non_empty=True) as temp_dir: @@ -736,7 +687,8 @@ def End(self, responses: flow_responses.Responses) -> None: self.handler.Handle(args, context=context) @db_test_lib.WithDatabase - def testValidatesParsersWereNotApplied(self, db: abstract_db.Database): + @fleetspeak_test_lib.WithFleetspeakConnector + def testValidatesParsersWereNotApplied(self, db: abstract_db.Database, _): context = _CreateContext(db) client_id = db_test_utils.InitializeClient(db) @@ -766,7 +718,8 @@ def testValidatesParsersWereNotApplied(self, db: abstract_db.Database): self.handler.Handle(args, context=context) @db_test_lib.WithDatabase - def testParsesArtifactCollectionResults(self, db: abstract_db.Database): + @fleetspeak_test_lib.WithFleetspeakConnector + def testParsesArtifactCollectionResults(self, db: abstract_db.Database, _): context = _CreateContext(db) with mock.patch.object(artifact_registry, "REGISTRY", @@ -820,7 +773,8 @@ def ParseResponse( self.assertEqual(response.stderr, b"4815162342") @db_test_lib.WithDatabase - def testReportsArtifactCollectionErrors(self, db: abstract_db.Database): + @fleetspeak_test_lib.WithFleetspeakConnector + def testReportsArtifactCollectionErrors(self, db: abstract_db.Database, _): context = _CreateContext(db) with mock.patch.object(artifact_registry, "REGISTRY", @@ -865,7 +819,8 @@ def ParseResponse( self.assertEqual(result.errors[0], "Lorem ipsum.") @db_test_lib.WithDatabase - def testUsesKnowledgebaseFromFlow(self, db: abstract_db.Database): + @fleetspeak_test_lib.WithFleetspeakConnector + def testUsesKnowledgebaseFromFlow(self, db: abstract_db.Database, _): context = _CreateContext(db) client_id = db_test_utils.InitializeClient(db) @@ -932,7 +887,8 @@ def ParseResponse( self.assertEqual(response.stderr.decode("utf-8"), "redox") @db_test_lib.WithDatabase - def testUsesCollectionTimeFiles(self, db: abstract_db.Database): + @fleetspeak_test_lib.WithFleetspeakConnector + def testUsesCollectionTimeFiles(self, db: abstract_db.Database, _): context = _CreateContext(db) client_id = db_test_utils.InitializeClient(db) @@ -1020,7 +976,8 @@ def ParseFile( self.assertEqual(response, b"OLD") @db_test_lib.WithDatabase - def testEmptyResults(self, db: abstract_db.Database): + @fleetspeak_test_lib.WithFleetspeakConnector + def testEmptyResults(self, db: abstract_db.Database, _): context = _CreateContext(db) client_id = db_test_utils.InitializeClient(db) @@ -1096,16 +1053,18 @@ def testScheduleFlow(self, db: abstract_db.Database): args = flow_plugin.ApiCreateFlowArgs( client_id=client_id, flow=flow_plugin.ApiFlow( - name=file.CollectSingleFile.__name__, - args=rdf_file_finder.CollectSingleFileArgs(path="/foo"), - runner_args=rdf_flow_runner.FlowRunnerArgs(cpu_limit=60))) + name=file.CollectFilesByKnownPath.__name__, + args=rdf_file_finder.CollectFilesByKnownPathArgs(paths=["/foo"]), + runner_args=rdf_flow_runner.FlowRunnerArgs(cpu_limit=60), + ), + ) sf = handler.Handle(args, context=context) self.assertEqual(sf.client_id, client_id) self.assertEqual(sf.creator, context.username) self.assertNotEmpty(sf.scheduled_flow_id) - self.assertEqual(sf.flow_name, file.CollectSingleFile.__name__) - self.assertEqual(sf.flow_args.path, "/foo") + self.assertEqual(sf.flow_name, file.CollectFilesByKnownPath.__name__) + self.assertEqual(sf.flow_args.paths, ["/foo"]) self.assertEqual(sf.runner_args.cpu_limit, 60) @db_test_lib.WithDatabase @@ -1119,26 +1078,41 @@ def testListScheduledFlows(self, db: abstract_db.Database): flow_plugin.ApiCreateFlowArgs( client_id=client_id1, flow=flow_plugin.ApiFlow( - name=file.CollectSingleFile.__name__, - args=rdf_file_finder.CollectSingleFileArgs(path="/foo"), - runner_args=rdf_flow_runner.FlowRunnerArgs(cpu_limit=60))), - context=context) + name=file.CollectFilesByKnownPath.__name__, + args=rdf_file_finder.CollectFilesByKnownPathArgs( + paths=["/foo"] + ), + runner_args=rdf_flow_runner.FlowRunnerArgs(cpu_limit=60), + ), + ), + context=context, + ) sf2 = handler.Handle( flow_plugin.ApiCreateFlowArgs( client_id=client_id1, flow=flow_plugin.ApiFlow( - name=file.CollectSingleFile.__name__, - args=rdf_file_finder.CollectSingleFileArgs(path="/foo"), - runner_args=rdf_flow_runner.FlowRunnerArgs(cpu_limit=60))), - context=context) + name=file.CollectFilesByKnownPath.__name__, + args=rdf_file_finder.CollectFilesByKnownPathArgs( + paths=["/foo"] + ), + runner_args=rdf_flow_runner.FlowRunnerArgs(cpu_limit=60), + ), + ), + context=context, + ) handler.Handle( flow_plugin.ApiCreateFlowArgs( client_id=client_id2, flow=flow_plugin.ApiFlow( - name=file.CollectSingleFile.__name__, - args=rdf_file_finder.CollectSingleFileArgs(path="/foo"), - runner_args=rdf_flow_runner.FlowRunnerArgs(cpu_limit=60))), - context=context) + name=file.CollectFilesByKnownPath.__name__, + args=rdf_file_finder.CollectFilesByKnownPathArgs( + paths=["/foo"] + ), + runner_args=rdf_flow_runner.FlowRunnerArgs(cpu_limit=60), + ), + ), + context=context, + ) handler = flow_plugin.ApiListScheduledFlowsHandler() args = flow_plugin.ApiListScheduledFlowsArgs( @@ -1157,18 +1131,28 @@ def testUnscheduleFlowRemovesScheduledFlow(self, db: abstract_db.Database): flow_plugin.ApiCreateFlowArgs( client_id=client_id, flow=flow_plugin.ApiFlow( - name=file.CollectSingleFile.__name__, - args=rdf_file_finder.CollectSingleFileArgs(path="/foo"), - runner_args=rdf_flow_runner.FlowRunnerArgs(cpu_limit=60))), - context=context) + name=file.CollectFilesByKnownPath.__name__, + args=rdf_file_finder.CollectFilesByKnownPathArgs( + paths=["/foo"] + ), + runner_args=rdf_flow_runner.FlowRunnerArgs(cpu_limit=60), + ), + ), + context=context, + ) sf2 = handler.Handle( flow_plugin.ApiCreateFlowArgs( client_id=client_id, flow=flow_plugin.ApiFlow( - name=file.CollectSingleFile.__name__, - args=rdf_file_finder.CollectSingleFileArgs(path="/foo"), - runner_args=rdf_flow_runner.FlowRunnerArgs(cpu_limit=60))), - context=context) + name=file.CollectFilesByKnownPath.__name__, + args=rdf_file_finder.CollectFilesByKnownPathArgs( + paths=["/foo"] + ), + runner_args=rdf_flow_runner.FlowRunnerArgs(cpu_limit=60), + ), + ), + context=context, + ) handler = flow_plugin.ApiUnscheduleFlowHandler() args = flow_plugin.ApiUnscheduleFlowArgs( diff --git a/grr/server/grr_response_server/gui/api_plugins/hunt.py b/grr/server/grr_response_server/gui/api_plugins/hunt.py index 9c92ecc6fd..62f07ac2a6 100644 --- a/grr/server/grr_response_server/gui/api_plugins/hunt.py +++ b/grr/server/grr_response_server/gui/api_plugins/hunt.py @@ -191,6 +191,7 @@ def InitFromHuntObject(self, self.name = "VariableGenericHunt" self.hunt_type = self.HuntType.VARIABLE self.state = str(hunt_obj.hunt_state) + self.state_reason = str(hunt_obj.hunt_state_reason) self.state_comment = str(hunt_obj.hunt_state_comment) self.crash_limit = hunt_obj.crash_limit self.client_limit = hunt_obj.client_limit @@ -262,7 +263,7 @@ def InitFromHuntObject(self, if hunt_obj.HasField("per_client_cpu_limit"): hra.per_client_cpu_limit = hunt_obj.per_client_cpu_limit - if hunt_obj.HasField("per_client_network_limit_bytes"): + if hunt_obj.HasField("per_client_network_bytes_limit"): hra.per_client_network_limit_bytes = ( hunt_obj.per_client_network_bytes_limit) @@ -1430,6 +1431,10 @@ def Handle(self, args, context=None): creator=context.username, ) + # Avoid messing with default values by checking presence first: + if hra.HasField("network_bytes_limit"): + hunt_obj.total_network_bytes_limit = hra.network_bytes_limit + if args.original_hunt and args.original_flow: raise ValueError( "A hunt can't be a copy of a flow and a hunt at the same time.") @@ -1499,32 +1504,41 @@ def Handle(self, args, context=None): data_store.REL_DB.UpdateHuntObject(hunt_id, **kw_args) hunt_obj = data_store.REL_DB.ReadHuntObject(hunt_id) except db.UnknownHuntError: - raise HuntNotFoundError("Hunt with id %s could not be found" % - args.hunt_id) + raise HuntNotFoundError( + "Hunt with id %s could not be found" % args.hunt_id + ) from None if args.HasField("state"): if args.state == ApiHunt.State.STARTED: if hunt_obj.hunt_state != hunt_obj.HuntState.PAUSED: raise HuntNotStartableError( - "Hunt can only be started from PAUSED state.") + "Hunt can only be started from PAUSED state." + ) hunt_obj = hunt.StartHunt(hunt_obj.hunt_id) elif args.state == ApiHunt.State.STOPPED: if hunt_obj.hunt_state not in [ - hunt_obj.HuntState.PAUSED, hunt_obj.HuntState.STARTED + hunt_obj.HuntState.PAUSED, + hunt_obj.HuntState.STARTED, ]: raise HuntNotStoppableError( - "Hunt can only be stopped from STARTED or " - "PAUSED states.") - hunt_obj = hunt.StopHunt(hunt_obj.hunt_id, reason=CANCELLED_BY_USER) + "Hunt can only be stopped from STARTED or PAUSED states." + ) + hunt_obj = hunt.StopHunt( + hunt_obj.hunt_id, + hunt_state_reason=rdf_hunt_objects.Hunt.HuntStateReason.TRIGGERED_BY_USER, + reason_comment=CANCELLED_BY_USER, + ) else: raise InvalidHuntStateError( - "Hunt's state can only be updated to STARTED or STOPPED") + "Hunt's state can only be updated to STARTED or STOPPED" + ) hunt_counters = data_store.REL_DB.ReadHuntCounters(hunt_obj.hunt_id) return ApiHunt().InitFromHuntObject( - hunt_obj, hunt_counters=hunt_counters, with_full_summary=True) + hunt_obj, hunt_counters=hunt_counters, with_full_summary=True + ) class ApiDeleteHuntArgs(rdf_structs.RDFProtoStruct): diff --git a/grr/server/grr_response_server/gui/api_plugins/hunt_test.py b/grr/server/grr_response_server/gui/api_plugins/hunt_test.py index f981d7f691..4ebaa08417 100644 --- a/grr/server/grr_response_server/gui/api_plugins/hunt_test.py +++ b/grr/server/grr_response_server/gui/api_plugins/hunt_test.py @@ -73,14 +73,33 @@ def setUp(self): def testQueueHuntRunnerArgumentIsNotRespected(self): args = hunt_plugin.ApiCreateHuntArgs( - flow_name=file_finder.FileFinder.__name__) + flow_name=file_finder.FileFinder.__name__ + ) args.hunt_runner_args.queue = "BLAH" result = self.handler.Handle(args, context=self.context) self.assertFalse(result.hunt_runner_args.HasField("queue")) + def testNetworkBytesLimitHuntRunnerArgumentIsRespected(self): + args = hunt_plugin.ApiCreateHuntArgs( + flow_name=file_finder.ClientFileFinder.__name__ + ) + args.hunt_runner_args.network_bytes_limit = 123 + + result = self.handler.Handle(args, context=self.context) + self.assertEqual(123, result.hunt_runner_args.network_bytes_limit) + + def testNetworkBytesLimitHuntRunnerArgumentDefaultRespected(self): + args = hunt_plugin.ApiCreateHuntArgs( + flow_name=file_finder.ClientFileFinder.__name__ + ) + + result = self.handler.Handle(args, context=self.context) + self.assertFalse(result.hunt_runner_args.HasField("network_bytes_limit")) + -class ApiListHuntsHandlerTest(api_test_lib.ApiCallHandlerTest, - hunt_test_lib.StandardHuntTestMixin): +class ApiListHuntsHandlerTest( + api_test_lib.ApiCallHandlerTest, hunt_test_lib.StandardHuntTestMixin +): """Test for ApiListHuntsHandler.""" def setUp(self): @@ -486,26 +505,32 @@ def testReturnsAllResultsOfAllTypes(self): hunt_id = self._RunHuntWithResults( client_count=5, results=[ - rdf_file_finder.CollectSingleFileResult(), - rdf_file_finder.FileFinderResult() - ]) + rdf_file_finder.CollectFilesByKnownPathResult(), + rdf_file_finder.FileFinderResult(), + ], + ) result = self.handler.Handle( hunt_plugin.ApiListHuntResultsArgs(hunt_id=hunt_id), context=self.context) - self.assertCountEqual([r.payload_type for r in result.items], [ - rdf_file_finder.CollectSingleFileResult.__name__, - rdf_file_finder.FileFinderResult.__name__ - ] * 5) + self.assertCountEqual( + [r.payload_type for r in result.items], + [ + rdf_file_finder.CollectFilesByKnownPathResult.__name__, + rdf_file_finder.FileFinderResult.__name__, + ] + * 5, + ) self.assertEqual(result.total_count, 10) def testCountsAllResultsWithAllTypes(self): hunt_id = self._RunHuntWithResults( client_count=5, results=[ - rdf_file_finder.CollectSingleFileResult(), - rdf_file_finder.FileFinderResult() - ]) + rdf_file_finder.CollectFilesByKnownPathResult(), + rdf_file_finder.FileFinderResult(), + ], + ) result = self.handler.Handle( hunt_plugin.ApiListHuntResultsArgs(hunt_id=hunt_id, count=3), context=self.context) @@ -517,9 +542,10 @@ def testReturnsAllResultsOfFilteredType(self): hunt_id = self._RunHuntWithResults( client_count=5, results=[ - rdf_file_finder.CollectSingleFileResult(), - rdf_file_finder.FileFinderResult() - ]) + rdf_file_finder.CollectFilesByKnownPathResult(), + rdf_file_finder.FileFinderResult(), + ], + ) result = self.handler.Handle( hunt_plugin.ApiListHuntResultsArgs( hunt_id=hunt_id, @@ -534,9 +560,10 @@ def testCountsAllResultsWithType(self): hunt_id = self._RunHuntWithResults( client_count=5, results=[ - rdf_file_finder.CollectSingleFileResult(), - rdf_file_finder.FileFinderResult() - ]) + rdf_file_finder.CollectFilesByKnownPathResult(), + rdf_file_finder.FileFinderResult(), + ], + ) result = self.handler.Handle( hunt_plugin.ApiListHuntResultsArgs( hunt_id=hunt_id, @@ -571,19 +598,26 @@ def testCountsAllResultsOfAllTypes(self): hunt_id = self._RunHuntWithResults( client_count=5, results=[ - rdf_file_finder.CollectSingleFileResult(), - rdf_file_finder.FileFinderResult() - ]) + rdf_file_finder.CollectFilesByKnownPathResult(), + rdf_file_finder.FileFinderResult(), + ], + ) result = self.handler.Handle( hunt_plugin.ApiListHuntResultsArgs(hunt_id=hunt_id), context=self.context) - self.assertCountEqual(result.items, [ - hunt_plugin.ApiTypeCount( - type=rdf_file_finder.CollectSingleFileResult.__name__, count=5), - hunt_plugin.ApiTypeCount( - type=rdf_file_finder.FileFinderResult.__name__, count=5), - ]) + self.assertCountEqual( + result.items, + [ + hunt_plugin.ApiTypeCount( + type=rdf_file_finder.CollectFilesByKnownPathResult.__name__, + count=5, + ), + hunt_plugin.ApiTypeCount( + type=rdf_file_finder.FileFinderResult.__name__, count=5 + ), + ], + ) class ApiListHuntOutputPluginLogsHandlerTest(api_test_lib.ApiCallHandlerTest, @@ -723,6 +757,10 @@ def testStopsHuntCorrectly(self): h = data_store.REL_DB.ReadHuntObject(self.hunt_id) self.assertEqual(h.hunt_state, h.HuntState.STOPPED) + self.assertEqual( + h.hunt_state_reason, + rdf_hunt_objects.Hunt.HuntStateReason.TRIGGERED_BY_USER, + ) self.assertEqual(h.hunt_state_comment, "Cancelled by user") def testRaisesWhenModifyingHuntInNonPausedState(self): diff --git a/grr/server/grr_response_server/gui/api_plugins/user.py b/grr/server/grr_response_server/gui/api_plugins/user.py index f27302c6ac..07face9ea2 100644 --- a/grr/server/grr_response_server/gui/api_plugins/user.py +++ b/grr/server/grr_response_server/gui/api_plugins/user.py @@ -1486,6 +1486,7 @@ class ApiListApproverSuggestionsHandler(api_call_handler_base.ApiCallHandler): def Handle(self, args, context=None): all_usernames = _GetAllUsernames() + all_usernames = sorted(set(all_usernames) - access_control.SYSTEM_USERS) usernames = [] if not args.username_query: diff --git a/grr/server/grr_response_server/gui/api_plugins/user_test.py b/grr/server/grr_response_server/gui/api_plugins/user_test.py index e1706e36cc..54d77e4d08 100644 --- a/grr/server/grr_response_server/gui/api_plugins/user_test.py +++ b/grr/server/grr_response_server/gui/api_plugins/user_test.py @@ -15,6 +15,7 @@ from grr_response_server import email_alerts from grr_response_server import flow from grr_response_server import notification +from grr_response_server.databases import db_test_utils from grr_response_server.flows import file from grr_response_server.gui import api_call_context from grr_response_server.gui import api_call_handler_base @@ -24,7 +25,6 @@ from grr_response_server.rdfvalues import cronjobs as rdf_cronjobs from grr_response_server.rdfvalues import flow_runner as rdf_flow_runner from grr_response_server.rdfvalues import objects as rdf_objects - from grr.test_lib import acl_test_lib from grr.test_lib import hunt_test_lib from grr.test_lib import notification_test_lib @@ -351,9 +351,10 @@ def testErrorDuringStartFlowDoesNotBubbleUpToApprovalApiCall(self): flow.ScheduleFlow( client_id=self.client_id, creator=self.context.username, - flow_name=file.CollectSingleFile.__name__, - flow_args=rdf_file_finder.CollectSingleFileArgs(path="/foo"), - runner_args=rdf_flow_runner.FlowRunnerArgs()) + flow_name=file.CollectFilesByKnownPath.__name__, + flow_args=rdf_file_finder.CollectFilesByKnownPathArgs(paths=["/foo"]), + runner_args=rdf_flow_runner.FlowRunnerArgs(), + ) with mock.patch.object( flow, "StartFlow", @@ -686,7 +687,8 @@ def setUp(self): cron_args = rdf_cronjobs.CreateCronJobArgs( frequency="1d", allow_overruns=False, - flow_name=file.CollectSingleFile.__name__) + flow_name=file.CollectFilesByKnownPath.__name__, + ) cron_id = cron_manager.CreateJob(cron_args=cron_args) self.handler = user_plugin.ApiCreateCronJobApprovalHandler() @@ -710,7 +712,8 @@ def testRendersRequestedCronJobApproval(self): cron_args = rdf_cronjobs.CreateCronJobArgs( frequency="1d", allow_overruns=False, - flow_name=file.CollectSingleFile.__name__) + flow_name=file.CollectFilesByKnownPath.__name__, + ) cron_job_id = cron_manager.CreateJob(cron_args=cron_args) self.RequestCronJobApproval( @@ -912,6 +915,18 @@ def testExcludesCurrentUser(self): self.assertLen(result.suggestions, 1) self.assertEqual(result.suggestions[0].username, "api_user_2") + def testExcludesSystemUsers(self): + non_system_username = db_test_utils.InitializeUser(data_store.REL_DB) + db_test_utils.InitializeUser(data_store.REL_DB, "GRRWorker") + db_test_utils.InitializeUser(data_store.REL_DB, "GRRCron") + + result = self._query("") + result_usernames = [_.username for _ in result.suggestions] + + self.assertIn(non_system_username, result_usernames) + self.assertNotIn("GRRWorker", result_usernames) + self.assertNotIn("GRRCron", result_usernames) + def testSuggestsMostRequestedUsers(self): client_id = self.SetupClient(0) self.RequestClientApproval(client_id, approver="sanchezmorty") diff --git a/grr/server/grr_response_server/gui/api_regression_test_lib.py b/grr/server/grr_response_server/gui/api_regression_test_lib.py index b0290f1883..0906e3940c 100644 --- a/grr/server/grr_response_server/gui/api_regression_test_lib.py +++ b/grr/server/grr_response_server/gui/api_regression_test_lib.py @@ -295,7 +295,13 @@ def Generate(self): except Exception as e: # pylint: disable=broad-except logging.exception(e) - json_sample_data = json.dumps(sample_data, sort_keys=True) + json_sample_data = json.dumps( + sample_data, + indent=2, + ensure_ascii=False, + separators=(",", ": "), + sort_keys=True, + ) sys.stdout.buffer.write(json_sample_data.encode("utf-8")) diff --git a/grr/server/grr_response_server/gui/gui_test_lib.py b/grr/server/grr_response_server/gui/gui_test_lib.py index d86485996f..420fc7f8d7 100644 --- a/grr/server/grr_response_server/gui/gui_test_lib.py +++ b/grr/server/grr_response_server/gui/gui_test_lib.py @@ -624,7 +624,7 @@ def MatSelect(self, target, label): raise ValueError("Only CSS selector is supported for material select.") self.WaitUntil(self.GetVisibleElement, target) self.driver.execute_script(f"$('{effective_selector}').click()") - self.Click(f"css=.mat-option-text:contains('{label}')") + self.Click(f"css=mat-option:contains('{label}')") def IsChecked(self, target): return self.WaitUntil(self.GetVisibleElement, target).is_selected() diff --git a/grr/server/grr_response_server/gui/selenium_tests/hunt_create_test.py b/grr/server/grr_response_server/gui/selenium_tests/hunt_create_test.py index d993cd0a44..4fe8e16d2d 100644 --- a/grr/server/grr_response_server/gui/selenium_tests/hunt_create_test.py +++ b/grr/server/grr_response_server/gui/selenium_tests/hunt_create_test.py @@ -420,7 +420,8 @@ def testOutputPluginsListEmptyWhenNoDefaultOutputPluginSet(self): def testDefaultOutputPluginIsCorrectlyAddedToThePluginsList(self): with test_lib.ConfigOverrider( - {"AdminUI.new_hunt_wizard.default_output_plugin": "DummyOutputPlugin"}): + {"AdminUI.new_hunt_wizard.default_output_plugins": "DummyOutputPlugin"} + ): self.Open("/legacy#main=ManageHunts") self.Click("css=button[name=NewHunt]") diff --git a/grr/server/grr_response_server/gui/selenium_tests/inspect_view_test.py b/grr/server/grr_response_server/gui/selenium_tests/inspect_view_test.py deleted file mode 100644 index 9c60b0f56e..0000000000 --- a/grr/server/grr_response_server/gui/selenium_tests/inspect_view_test.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python -"""Test the inspect interface.""" - -from absl import app - -from grr_response_core.lib import rdfvalue -from grr_response_server import data_store -from grr_response_server import flow -from grr_response_server.flows.general import discovery as flow_discovery -from grr_response_server.flows.general import processes -from grr_response_server.gui import gui_test_lib -from grr_response_server.rdfvalues import flow_objects as rdf_flow_objects -from grr.test_lib import test_lib - - -class TestInspectViewBase(gui_test_lib.GRRSeleniumTest): - pass - - -class TestClientLoadView(TestInspectViewBase): - """Tests for ClientLoadView.""" - - def setUp(self): - super().setUp() - self.client_id = self.SetupClient(0) - - def CreateLeasedClientRequest(self, client_id=None): - flow.StartFlow(client_id=client_id, flow_cls=processes.ListProcesses) - client_messages = data_store.REL_DB.LeaseClientActionRequests( - client_id, lease_time=rdfvalue.Duration.From(10000, rdfvalue.SECONDS)) - self.assertNotEmpty(client_messages) - - def testNoClientActionIsDisplayed(self): - self.RequestAndGrantClientApproval(self.client_id) - - self.Open("/legacy#/clients/%s/load-stats" % self.client_id) - self.WaitUntil(self.IsTextPresent, "No actions currently in progress.") - - def testClientActionIsDisplayedWhenItReceiveByTheClient(self): - self.RequestAndGrantClientApproval(self.client_id) - self.CreateLeasedClientRequest(client_id=self.client_id) - - self.Open("/legacy#/clients/%s/load-stats" % self.client_id) - self.WaitUntil(self.IsTextPresent, processes.ListProcesses.__name__) - self.WaitUntil(self.IsTextPresent, "Leased until") - - -class TestDebugClientRequestsView(TestInspectViewBase): - """Test the inspect interface.""" - - def testInspect(self): - """Test the inspect UI.""" - client_id = self.SetupClient(0) - - self.RequestAndGrantClientApproval(client_id) - - flow_id = flow.StartFlow( - client_id=client_id, flow_cls=flow_discovery.Interrogate) - status = rdf_flow_objects.FlowStatus( - client_id=client_id, flow_id=flow_id, request_id=1, response_id=2) - data_store.REL_DB.WriteFlowResponses([status]) - - self.Open("/legacy#/clients/%s/debug-requests" % client_id) - - # Check that the we can see both requests and responses. - self.WaitUntil(self.IsTextPresent, "GetPlatformInfo") - self.WaitUntil(self.IsTextPresent, "GetConfig") - self.WaitUntil(self.IsTextPresent, "EnumerateInterfaces") - - self.WaitUntil(self.IsTextPresent, "STATUS") - - -if __name__ == "__main__": - app.run(test_lib.main) diff --git a/grr/server/grr_response_server/gui/selenium_tests/v2/approval_page_test.py b/grr/server/grr_response_server/gui/selenium_tests/v2/approval_page_test.py index 82bf5e566b..70daa7820a 100644 --- a/grr/server/grr_response_server/gui/selenium_tests/v2/approval_page_test.py +++ b/grr/server/grr_response_server/gui/selenium_tests/v2/approval_page_test.py @@ -61,9 +61,10 @@ def testScheduledFlowsAreShown(self): flow.ScheduleFlow( client_id=client_id, creator="requestrick", - flow_name=file.CollectSingleFile.__name__, - flow_args=rdf_file_finder.CollectSingleFileArgs(path="/foo"), - runner_args=rdf_flow_runner.FlowRunnerArgs()) + flow_name=file.CollectFilesByKnownPath.__name__, + flow_args=rdf_file_finder.CollectFilesByKnownPathArgs(paths=["/foo"]), + runner_args=rdf_flow_runner.FlowRunnerArgs(), + ) approval_id = self.RequestClientApproval( client_id, @@ -75,7 +76,7 @@ def testScheduledFlowsAreShown(self): f"/v2/clients/{client_id}/users/requestrick/approvals/{approval_id}") # Change to pretty display name as soon as ScheduledFlowList uses these. - self.WaitUntil(self.IsTextPresent, "CollectSingleFile") + self.WaitUntil(self.IsTextPresent, "CollectFilesByKnownPath") def testBackButtonNavigatesToOldUi(self): client_id = self.SetupClient(0) diff --git a/grr/server/grr_response_server/gui/selenium_tests/v2/client_test.py b/grr/server/grr_response_server/gui/selenium_tests/v2/client_test.py index c843166ab5..ba1c2a2c02 100644 --- a/grr/server/grr_response_server/gui/selenium_tests/v2/client_test.py +++ b/grr/server/grr_response_server/gui/selenium_tests/v2/client_test.py @@ -18,7 +18,7 @@ def testLabelsAreShownAndCanBeRemoved(self): self.WaitUntil(self.IsTextPresent, "mylabel") self.WaitUntil(self.IsTextPresent, "foobar") - self.Click("css=:contains('mylabel') > [matchipremove]") + self.Click("css=mat-chip:contains('mylabel') [matchipremove]") self.WaitUntilNot(self.IsTextPresent, "mylabel") self.assertEqual( diff --git a/grr/server/grr_response_server/gui/selenium_tests/v2/file_test.py b/grr/server/grr_response_server/gui/selenium_tests/v2/file_test.py index d3ae7852a9..9b2ab7d6b9 100644 --- a/grr/server/grr_response_server/gui/selenium_tests/v2/file_test.py +++ b/grr/server/grr_response_server/gui/selenium_tests/v2/file_test.py @@ -19,183 +19,6 @@ from grr.test_lib import test_lib -class CollectSingleFileTest(gui_test_lib.GRRSeleniumTest): - """Tests the search UI.""" - - def _GenSampleResult(self, - use_ntfs: bool = False - ) -> rdf_file_finder.CollectSingleFileResult: - if use_ntfs: - pathspec = rdf_paths.PathSpec.NTFS(path="/etc/hosts") - else: - pathspec = rdf_paths.PathSpec.OS(path="/etc/hosts") - - return rdf_file_finder.CollectSingleFileResult( - stat=rdf_client_fs.StatEntry( - pathspec=pathspec, - st_mode=33184, - st_size=4242, - st_atime=1336469177, - st_mtime=1336129892, - st_ctime=1336129892, - ), - hash=rdf_crypto.Hash( - sha256=binascii.unhexlify( - "9e8dc93e150021bb4752029ebbff51394aa36f069cf19901578e4f06017acdb5" - ), - sha1=binascii.unhexlify("6dd6bee591dfcb6d75eb705405302c3eab65e21a"), - md5=binascii.unhexlify("8b0a15eefe63fd41f8dc9dee01c5cf9a"))) - - def setUp(self): - super().setUp() - self.client_id = self.SetupClient(0) - self.RequestAndGrantClientApproval(self.client_id) - - def testCorrectlyDisplaysInProgressStatus(self): - flow_args = rdf_file_finder.CollectSingleFileArgs(path="/etc/hosts") - flow_test_lib.StartFlow( - file.CollectSingleFile, - creator=self.test_username, - client_id=self.client_id, - flow_args=flow_args) - - with flow_test_lib.FlowProgressOverride( - file.CollectSingleFile, - rdf_file_finder.CollectSingleFileProgress( - status=rdf_file_finder.CollectSingleFileProgress.Status.IN_PROGRESS) - ): - self.Open(f"/v2/clients/{self.client_id}") - self.WaitUntil(self.IsElementPresent, - "css=.flow-title:contains('File content')") - self.WaitUntil( - self.IsElementPresent, - "css=collect-single-file-details .requested-path:contains('/etc/hosts')" - ) - self.WaitUntilNot(self.IsElementPresent, - "css=collect-single-file-details .collected-result") - - def testCorrectlyDisplaysCollectedResult(self): - flow_args = rdf_file_finder.CollectSingleFileArgs(path="/etc/hosts") - flow_test_lib.StartFlow( - file.CollectSingleFile, - creator=self.test_username, - client_id=self.client_id, - flow_args=flow_args) - - with flow_test_lib.FlowProgressOverride( - file.CollectSingleFile, - rdf_file_finder.CollectSingleFileProgress( - status=rdf_file_finder.CollectSingleFileProgress.Status.COLLECTED, - result=self._GenSampleResult())): - - self.Open(f"/v2/clients/{self.client_id}") - self.WaitUntil( - self.IsElementPresent, - "css=collect-single-file-details .collected-result:contains('/etc/hosts')" - ) - self.WaitUntil( - self.IsElementPresent, - "css=collect-single-file-details .collected-result:contains('4.14 KiB')" - ) - self.WaitUntilNot(self.IsElementPresent, - "css=collect-single-file-details .requested-path") - - def testCorrectlyDisplaysNonStandardPathTypeNote(self): - flow_args = rdf_file_finder.CollectSingleFileArgs(path="/etc/hosts") - flow_test_lib.StartFlow( - file.CollectSingleFile, - creator=self.test_username, - client_id=self.client_id, - flow_args=flow_args) - - with flow_test_lib.FlowProgressOverride( - file.CollectSingleFile, - rdf_file_finder.CollectSingleFileProgress( - status=rdf_file_finder.CollectSingleFileProgress.Status.COLLECTED, - result=self._GenSampleResult(use_ntfs=True))): - - self.Open(f"/v2/clients/{self.client_id}") - self.WaitUntil( - self.IsElementPresent, - "css=collect-single-file-details .collected-result:contains('/etc/hosts')" - ) - self.WaitUntil( - self.IsElementPresent, - "css=collect-single-file-details .path-type-note:contains(" - "'File was fetched by parsing the raw disk image with libfsntfs')") - - def testCorrectlyDisplaysDownloadButtonOnSuccess(self): - flow_args = rdf_file_finder.CollectSingleFileArgs(path="/etc/hosts") - flow_id = flow_test_lib.StartFlow( - file.CollectSingleFile, - creator=self.test_username, - client_id=self.client_id, - flow_args=flow_args) - - self.Open(f"/v2/clients/{self.client_id}") - self.WaitUntil( - self.IsElementPresent, - "css=collect-single-file-details .requested-path:contains('/etc/hosts')" - ) - self.WaitUntilNot(self.IsElementPresent, - "css=a[mat-stroked-button]:contains('Download')") - - flow_test_lib.MarkFlowAsFinished(self.client_id, flow_id) - - with flow_test_lib.FlowProgressOverride( - file.CollectSingleFile, - rdf_file_finder.CollectSingleFileProgress( - status=rdf_file_finder.CollectSingleFileProgress.Status.COLLECTED, - result=self._GenSampleResult())): - with flow_test_lib.FlowResultMetadataOverride( - file.CollectSingleFile, - rdf_flow_objects.FlowResultMetadata( - is_metadata_set=True, - num_results_per_type_tag=[ - rdf_flow_objects.FlowResultCount( - type=rdf_file_finder.CollectSingleFileResult.__name__, - count=1) - ])): - self.WaitUntil(self.IsElementPresent, - "css=a[mat-stroked-button]:contains('Download')") - - def testCorrectlyDisplaysNotFoundResult(self): - flow_args = rdf_file_finder.CollectSingleFileArgs(path="/etc/hosts") - flow_id = flow_test_lib.StartFlow( - file.CollectSingleFile, - creator=self.test_username, - client_id=self.client_id, - flow_args=flow_args) - flow_test_lib.MarkFlowAsFailed(self.client_id, flow_id) - - with flow_test_lib.FlowProgressOverride( - file.CollectSingleFile, - rdf_file_finder.CollectSingleFileProgress( - status=rdf_file_finder.CollectSingleFileProgress.Status.NOT_FOUND)): - self.Open(f"/v2/clients/{self.client_id}") - self.WaitUntil(self.IsElementPresent, "css=.flow-status mat-icon.error") - self.WaitUntil(self.IsElementPresent, - "css=flow-details :contains('File not found')") - - def testCorrectlyDisplaysError(self): - flow_args = rdf_file_finder.CollectSingleFileArgs(path="/etc/hosts") - flow_id = flow_test_lib.StartFlow( - file.CollectSingleFile, - creator=self.test_username, - client_id=self.client_id, - flow_args=flow_args) - flow_test_lib.MarkFlowAsFailed(self.client_id, flow_id) - - with flow_test_lib.FlowProgressOverride( - file.CollectSingleFile, - rdf_file_finder.CollectSingleFileProgress( - status=rdf_file_finder.CollectSingleFileProgress.Status.FAILED, - error_description="Something went wrong")): - self.Open(f"/v2/clients/{self.client_id}") - self.WaitUntil(self.IsElementPresent, - "css=flow-details :contains('Something went wrong')") - - class CollectFilesByKnownPathTest(gui_test_lib.GRRSeleniumTest): """Tests the CollectFilesByKnownPath Flow.""" @@ -278,7 +101,8 @@ def testDisplaysCollectedResult(self): self.WaitUntil( self.IsElementPresent, - "css=collect-files-by-known-path-details .mat-tab-label-content:contains('3 successful file collections')" + "css=collect-files-by-known-path-details" + " [role=tab]:contains('3 successful file collections')", ) for i in range(3): self.WaitUntil( @@ -308,7 +132,8 @@ def testDisplaysFailedResult(self): self.WaitUntil( self.IsElementPresent, - "css=collect-files-by-known-path-details .mat-tab-label-content:contains('2 errors')" + "css=collect-files-by-known-path-details" + " [role=tab]:contains('2 errors')", ) for i in range(2): self.WaitUntil( @@ -346,7 +171,8 @@ def testDisplaysSuccessAndFailedResultAndWarning(self): self.WaitUntil( self.IsElementPresent, - "css=collect-files-by-known-path-details .mat-tab-label-content:contains('1 successful file collection')" + "css=collect-files-by-known-path-details" + " [role=tab]:contains('1 successful file collection')", ) self.WaitUntil( self.IsElementPresent, @@ -354,10 +180,12 @@ def testDisplaysSuccessAndFailedResultAndWarning(self): self.WaitUntil( self.IsElementPresent, - "css=collect-files-by-known-path-details .mat-tab-label-content:contains('1 error')" + "css=collect-files-by-known-path-details" + " [role=tab]:contains('1 error')", ) self.Click( - "css=collect-files-by-known-path-details .mat-tab-label-content:contains('1 error')" + "css=collect-files-by-known-path-details" + " [role=tab]:contains('1 error')" ) self.WaitUntil( self.IsElementPresent, @@ -427,7 +255,9 @@ def testFlowArgumentForm(self): self.Click('css=flow-form button:contains("Collect files")') self.Click( - 'css=.mat-menu-panel button:contains("Collect files from exact paths")') + 'css=.mat-mdc-menu-panel button:contains("Collect files from exact' + ' paths")' + ) element = self.WaitUntil(self.GetVisibleElement, "css=flow-args-form textarea[name=paths]") @@ -517,7 +347,7 @@ def testNavigateToFileInFilesView(self): self.WaitUntilContains("st_size 14 B", self.GetText, "css=app-stat-view") - self.Click("css=a.mat-tab-link:contains('Text content')") + self.Click("css=a[role=tab]:contains('Text content')") self.WaitUntilContains("text", self.GetCurrentUrlPath) diff --git a/grr/server/grr_response_server/gui/selenium_tests/v2/flow_test.py b/grr/server/grr_response_server/gui/selenium_tests/v2/flow_test.py index 6eb7702f72..70070b03a5 100644 --- a/grr/server/grr_response_server/gui/selenium_tests/v2/flow_test.py +++ b/grr/server/grr_response_server/gui/selenium_tests/v2/flow_test.py @@ -87,7 +87,8 @@ def testCanScheduleFlowWithoutApproval(self): self.Click('css=flow-form button:contains("Collect files")') self.Click( - 'css=.mat-menu-panel button:contains("Collect files by search criteria")' + 'css=.mat-mdc-menu-panel button:contains("Collect files by search' + ' criteria")' ) self.Type('css=flow-args-form app-glob-expression-input input', '/foo/test') @@ -153,7 +154,8 @@ def testCanCreateFlowAfterGrantedApproval(self): self.Click('css=flow-form button:contains("Collect files")') self.Click( - 'css=.mat-menu-panel button:contains("Collect files by search criteria")' + 'css=.mat-mdc-menu-panel button:contains("Collect files by search' + ' criteria")' ) self.Type('css=flow-args-form app-glob-expression-input input', '/foo/test') @@ -188,7 +190,8 @@ def testCanDuplicateFlow(self): self.Click('css=flow-form button:contains("Collect files")') self.Click( - 'css=.mat-menu-panel button:contains("Collect files by search criteria")' + 'css=.mat-mdc-menu-panel button:contains("Collect files by search' + ' criteria")' ) self.Type('css=flow-args-form app-glob-expression-input input', '/foo/test') self.Click('css=flow-form button:contains("Start")') @@ -196,7 +199,7 @@ def testCanDuplicateFlow(self): self.WaitUntilContains('/foo/test', self.GetText, 'css=flow-details') self.Click('css=flow-details button[aria-label="Flow menu"]') - self.Click('css=.mat-menu-panel button:contains("Duplicate flow")') + self.Click('css=.mat-mdc-menu-panel button:contains("Duplicate flow")') self.Click('css=flow-form button:contains("Start")') self.WaitUntilContains('/foo/test', self.GetText, @@ -229,7 +232,8 @@ def testCollectMultipleFilesFlow(self): self.Click('css=flow-form button:contains("Collect files")') self.Click( - 'css=.mat-menu-panel button:contains("Collect files by search criteria")' + 'css=.mat-mdc-menu-panel button:contains("Collect files by search' + ' criteria")' ) self.Type( @@ -334,7 +338,7 @@ def testScheduleArtifactCollectorFlow(self): # Type whole artifact name except last letter into autocomplete. self.Type('css=flow-args-form input[name=artifactName]', 'FakeFileArtifac') - self.Click('css=.mat-option:contains("FakeFileArtifact")') + self.Click('css=[role=option]:contains("FakeFileArtifact")') self.WaitUntilContains('Collects file', self.GetText, 'css=flow-args-form') self.WaitUntilContains('/notafile', self.GetText, 'css=flow-args-form') @@ -370,7 +374,7 @@ def testScheduleArtifactCollectorFlowWithDefaultArtifacts(self): # Type whole artifact name except last letter into autocomplete. self.Type('css=flow-args-form input[name=artifactName]', 'LinuxHardwareInf') - self.Click('css=.mat-option:contains("LinuxHardwareInfo")') + self.Click('css=[role=option]:contains("LinuxHardwareInfo")') self.Click('css=flow-form button:contains("Schedule")') def GetFirstScheduledFlow(): @@ -542,7 +546,8 @@ def testCanCreateFlowWithApprovalsDisabled(self): self.Click('css=flow-form button:contains("Collect files")') self.Click( - 'css=.mat-menu-panel button:contains("Collect files by search criteria")' + 'css=.mat-mdc-menu-panel button:contains("Collect files by search' + ' criteria")' ) self.Type('css=flow-args-form app-glob-expression-input input', '/foo/test') diff --git a/grr/server/grr_response_server/gui/selenium_tests/v2/hunt_page_test.py b/grr/server/grr_response_server/gui/selenium_tests/v2/hunt_page_test.py index b1f0b2f228..1ba6b2ff39 100644 --- a/grr/server/grr_response_server/gui/selenium_tests/v2/hunt_page_test.py +++ b/grr/server/grr_response_server/gui/selenium_tests/v2/hunt_page_test.py @@ -153,7 +153,8 @@ def testDisplaysHuntInformation(self): self.WaitUntil( self.IsElementPresent, - "css=app-hunt-results .mat-tab-label-active:contains('Errors')", + "css=app-hunt-results" + " [role=tab][aria-selected=true]:contains('Errors')", ) self.WaitUntil( self.IsElementPresent, @@ -179,9 +180,10 @@ def testDisplaysHuntInformation(self): self.WaitUntil( self.IsElementPresent, - "css=app-hunt-results .mat-tab-label:contains('File Finder')", + "css=app-hunt-results [role=tab]:contains('File Finder')", ) - self.Click("css=app-hunt-results .mat-tab-label:contains('File Finder')") + self.Click("css=app-hunt-results [role=tab]:contains('File Finder')") + self.WaitUntil( self.IsElementPresent, f"css=app-hunt-results mat-row:contains('{client_ids[1]}')", @@ -248,7 +250,7 @@ def testStoptHunt(self): self.Click("css=button[name=cancel-button]") self.WaitUntil( - self.IsElementPresent, "css=.hunt-overview:contains('cancelled')" + self.IsElementPresent, "css=.hunt-overview:contains('Cancelled')" ) hunts = _ListHunts(self.test_username) diff --git a/grr/server/grr_response_server/gui/selenium_tests/v2/list_directory_test.py b/grr/server/grr_response_server/gui/selenium_tests/v2/list_directory_test.py index d0f0f276cc..d8ec0c9978 100644 --- a/grr/server/grr_response_server/gui/selenium_tests/v2/list_directory_test.py +++ b/grr/server/grr_response_server/gui/selenium_tests/v2/list_directory_test.py @@ -172,13 +172,13 @@ def testPaginationNavigation(self): self.assertEqual(self.GetCssCount("css=td.path:contains('file')"), 10) # Navigation works in both top and bottom paginators. - self.Click("css=.top-paginator .mat-paginator-navigation-last") + self.Click("css=.top-paginator .mat-mdc-paginator-navigation-last") for i in range(10, 15): self.WaitUntil(self.IsElementPresent, f"css=td:contains('file{i}')") self.assertEqual(self.GetCssCount("css=td.path:contains('file')"), 5) self.ScrollToBottom() - self.Click("css=.bottom-paginator .mat-paginator-navigation-previous") + self.Click("css=.bottom-paginator .mat-mdc-paginator-navigation-previous") for i in range(10): self.WaitUntil(self.IsElementPresent, f"css=td:contains('file{i}')") self.assertEqual(self.GetCssCount("css=td.path:contains('file')"), 10) diff --git a/grr/server/grr_response_server/gui/selenium_tests/v2/netstat_test.py b/grr/server/grr_response_server/gui/selenium_tests/v2/netstat_test.py index d7a261c858..5787e66974 100644 --- a/grr/server/grr_response_server/gui/selenium_tests/v2/netstat_test.py +++ b/grr/server/grr_response_server/gui/selenium_tests/v2/netstat_test.py @@ -89,13 +89,13 @@ def testPaginationNavigation(self): self.assertEqual(self.GetCssCount("css=td:contains('process')"), 10) # Navigation works in both top and bottom paginators. - self.Click("css=.top-paginator .mat-paginator-navigation-last") + self.Click("css=.top-paginator .mat-mdc-paginator-navigation-last") for i in range(10, 15): self.WaitUntil(self.IsElementPresent, f"css=td:contains('process{i}')") self.assertEqual(self.GetCssCount("css=td:contains('process')"), 5) self.ScrollToBottom() - self.Click("css=.bottom-paginator .mat-paginator-navigation-previous") + self.Click("css=.bottom-paginator .mat-mdc-paginator-navigation-previous") for i in range(10): self.WaitUntil(self.IsElementPresent, f"css=td:contains('process{i}')") self.assertEqual(self.GetCssCount("css=td:contains('process')"), 10) diff --git a/grr/server/grr_response_server/gui/static/angular-components/cron/new-cron-job-wizard/form-directive.js b/grr/server/grr_response_server/gui/static/angular-components/cron/new-cron-job-wizard/form-directive.js index 4aa3206415..2531cfd6c4 100644 --- a/grr/server/grr_response_server/gui/static/angular-components/cron/new-cron-job-wizard/form-directive.js +++ b/grr/server/grr_response_server/gui/static/angular-components/cron/new-cron-job-wizard/form-directive.js @@ -3,7 +3,7 @@ goog.module.declareLegacyNamespace(); const apiService = goog.requireType('grrUi.core.apiService'); const reflectionService = goog.requireType('grrUi.core.reflectionService'); -const {DEFAULT_PLUGIN_URL} = goog.require('grrUi.hunt.newHuntWizard.formDirective'); +const {DEFAULT_PLUGINS_URL} = goog.require('grrUi.hunt.newHuntWizard.formDirective'); @@ -35,7 +35,7 @@ const FormController = class { /** @private {?string} */ this.defaultOutputPluginName; - this.grrApiService_.get(DEFAULT_PLUGIN_URL) + this.grrApiService_.get(DEFAULT_PLUGINS_URL) .then(function(response) { if (angular.isDefined(response['data']['value'])) { this.defaultOutputPluginName = response['data']['value']['value']; diff --git a/grr/server/grr_response_server/gui/static/angular-components/docs/api-docs-examples.json b/grr/server/grr_response_server/gui/static/angular-components/docs/api-docs-examples.json index efb9f7d4a7..1a1185f297 100644 --- a/grr/server/grr_response_server/gui/static/angular-components/docs/api-docs-examples.json +++ b/grr/server/grr_response_server/gui/static/angular-components/docs/api-docs-examples.json @@ -4,7 +4,10 @@ "api_method": "CancelFlow", "method": "POST", "response": { - "args": { "type": "ListProcessesArgs", "value": {} }, + "args": { + "type": "ListProcessesArgs", + "value": {} + }, "client_id": "C.1000000000000000", "creator": "", "error_description": "Cancelled by user", @@ -19,8 +22,14 @@ "type": "ClientURN", "value": "aff4:/C.1000000000000000" }, - "flow_name": { "type": "unicode", "value": "ListProcesses" }, - "notify_to_user": { "type": "bool", "value": false } + "flow_name": { + "type": "unicode", + "value": "ListProcesses" + }, + "notify_to_user": { + "type": "bool", + "value": false + } } }, "started_at": 42000000, @@ -58,15 +67,26 @@ { "type": "ApiTypeCount", "value": { - "count": { "type": "long", "value": 1 }, - "type": { "type": "unicode", "value": "RDFString" } + "count": { + "type": "long", + "value": 1 + }, + "type": { + "type": "unicode", + "value": "RDFString" + } } } ] }, "test_class": "ApiCountHuntResultsByTypeRegressionTest_http_v1", "type_stripped_response": { - "items": [{ "count": 1, "type": "RDFString" }] + "items": [ + { + "count": 1, + "type": "RDFString" + } + ] }, "url": "/api/hunts/H:123456/result-counts" } @@ -77,17 +97,30 @@ "method": "POST", "request_payload": { "approval": { - "email_cc_addresses": ["test@example.com"], - "notified_users": ["approver1", "approver2"], + "email_cc_addresses": [ + "test@example.com" + ], + "notified_users": [ + "approver1", + "approver2" + ], "reason": "really important reason!" } }, "response": { "type": "ApiClientApproval", "value": { - "approvers": [{ "type": "unicode", "value": "api_test_user" }], + "approvers": [ + { + "type": "unicode", + "value": "api_test_user" + } + ], "email_cc_addresses": [ - { "type": "unicode", "value": "test@example.com" } + { + "type": "unicode", + "value": "test@example.com" + } ], "email_message_id": { "type": "unicode", @@ -97,31 +130,67 @@ "type": "RDFDatetime", "value": 2419326000000 }, - "id": { "type": "unicode", "value": "approval:112233" }, - "is_valid": { "type": "bool", "value": false }, + "id": { + "type": "unicode", + "value": "approval:112233" + }, + "is_valid": { + "type": "bool", + "value": false + }, "is_valid_message": { "type": "unicode", "value": "Need at least 1 additional approver for access." }, "notified_users": [ - { "type": "unicode", "value": "approver1" }, - { "type": "unicode", "value": "approver2" } + { + "type": "unicode", + "value": "approver1" + }, + { + "type": "unicode", + "value": "approver2" + } ], - "reason": { "type": "unicode", "value": "really important reason!" }, - "requestor": { "type": "unicode", "value": "api_test_user" }, + "reason": { + "type": "unicode", + "value": "really important reason!" + }, + "requestor": { + "type": "unicode", + "value": "api_test_user" + }, "subject": { "type": "ApiClient", "value": { - "age": { "type": "RDFDatetime", "value": 42000000 }, + "age": { + "type": "RDFDatetime", + "value": 42000000 + }, "agent_info": { "type": "ClientInformation", "value": { - "build_time": { "type": "unicode", "value": "1980-01-01" }, - "client_name": { "type": "unicode", "value": "GRR Monitor" }, - "client_version": { "type": "long", "value": 1234 }, + "build_time": { + "type": "unicode", + "value": "1980-01-01" + }, + "client_name": { + "type": "unicode", + "value": "GRR Monitor" + }, + "client_version": { + "type": "long", + "value": 1234 + }, "labels": [ - { "type": "unicode", "value": "label1" }, - { "type": "unicode", "value": "label2" } + { + "type": "unicode", + "value": "label1" + }, + { + "type": "unicode", + "value": "label2" + } ] } }, @@ -129,7 +198,6 @@ "type": "ApiClientId", "value": "C.1000000000000000" }, - "fleetspeak_enabled": { "type": "bool", "value": false }, "hardware_info": { "type": "HardwareInfo", "value": { @@ -175,55 +243,100 @@ } } ], - "ifname": { "type": "unicode", "value": "if0" } + "ifname": { + "type": "unicode", + "value": "if0" + } } }, { "type": "Interface", "value": { - "ifname": { "type": "unicode", "value": "if1" }, - "mac_address": { "type": "MacAddress", "value": "qrvM3e4A" } + "ifname": { + "type": "unicode", + "value": "if1" + }, + "mac_address": { + "type": "MacAddress", + "value": "qrvM3e4A" + } } }, { "type": "Interface", "value": { - "ifname": { "type": "unicode", "value": "if2" }, - "mac_address": { "type": "MacAddress", "value": "u8zd7v8A" } + "ifname": { + "type": "unicode", + "value": "if2" + }, + "mac_address": { + "type": "MacAddress", + "value": "u8zd7v8A" + } } } ], "knowledge_base": { "type": "KnowledgeBase", "value": { - "fqdn": { "type": "unicode", "value": "Host-0.example.com" }, - "os": { "type": "unicode", "value": "Linux" }, + "fqdn": { + "type": "unicode", + "value": "Host-0.example.com" + }, + "os": { + "type": "unicode", + "value": "Linux" + }, "users": [ { "type": "User", "value": { - "username": { "type": "unicode", "value": "user1" } + "username": { + "type": "unicode", + "value": "user1" + } } }, { "type": "User", "value": { - "username": { "type": "unicode", "value": "user2" } + "username": { + "type": "unicode", + "value": "user2" + } } } ] } }, "labels": [], - "last_seen_at": { "type": "RDFDatetime", "value": 42000000 }, + "last_seen_at": { + "type": "RDFDatetime", + "value": 42000000 + }, "os_info": { "type": "Uname", "value": { - "fqdn": { "type": "unicode", "value": "Host-0.example.com" }, - "kernel": { "type": "unicode", "value": "4.0.0" }, - "machine": { "type": "unicode", "value": "x86_64" }, - "system": { "type": "unicode", "value": "Linux" }, - "version": { "type": "unicode", "value": "buster/sid" } + "fqdn": { + "type": "unicode", + "value": "Host-0.example.com" + }, + "kernel": { + "type": "unicode", + "value": "4.0.0" + }, + "machine": { + "type": "unicode", + "value": "x86_64" + }, + "system": { + "type": "unicode", + "value": "Linux" + }, + "version": { + "type": "unicode", + "value": "buster/sid" + } } }, "urn": { @@ -234,13 +347,19 @@ { "type": "User", "value": { - "username": { "type": "unicode", "value": "user1" } + "username": { + "type": "unicode", + "value": "user1" + } } }, { "type": "User", "value": { - "username": { "type": "unicode", "value": "user2" } + "username": { + "type": "unicode", + "value": "user2" + } } } ] @@ -250,14 +369,21 @@ }, "test_class": "ApiCreateClientApprovalHandlerRegressionTest_http_v1", "type_stripped_response": { - "approvers": ["api_test_user"], - "email_cc_addresses": ["test@example.com"], + "approvers": [ + "api_test_user" + ], + "email_cc_addresses": [ + "test@example.com" + ], "email_message_id": "", "expiration_time_us": 2419326000000, "id": "approval:112233", "is_valid": false, "is_valid_message": "Need at least 1 additional approver for access.", - "notified_users": ["approver1", "approver2"], + "notified_users": [ + "approver1", + "approver2" + ], "reason": "really important reason!", "requestor": "api_test_user", "subject": { @@ -266,10 +392,12 @@ "build_time": "1980-01-01", "client_name": "GRR Monitor", "client_version": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "client_id": "C.1000000000000000", - "fleetspeak_enabled": false, "hardware_info": { "bios_version": "Bios-Version-0", "system_manufacturer": "System-Manufacturer-0" @@ -277,7 +405,10 @@ "interfaces": [ { "addresses": [ - { "address_type": "INET", "packed_bytes": "wKgAAA==" }, + { + "address_type": "INET", + "packed_bytes": "wKgAAA==" + }, { "address_type": "INET6", "packed_bytes": "IAGrzQAAAAAAAAAAAAAAAA==" @@ -285,13 +416,26 @@ ], "ifname": "if0" }, - { "ifname": "if1", "mac_address": "qrvM3e4A" }, - { "ifname": "if2", "mac_address": "u8zd7v8A" } + { + "ifname": "if1", + "mac_address": "qrvM3e4A" + }, + { + "ifname": "if2", + "mac_address": "u8zd7v8A" + } ], "knowledge_base": { "fqdn": "Host-0.example.com", "os": "Linux", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] }, "labels": [], "last_seen_at": 42000000, @@ -303,7 +447,14 @@ "version": "buster/sid" }, "urn": "aff4:/C.1000000000000000", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] } }, "url": "/api/users/me/approvals/client/C.1000000000000000" @@ -315,38 +466,72 @@ "method": "POST", "request_payload": { "approval": { - "email_cc_addresses": ["test@example.com"], - "notified_users": ["approver1", "approver2"], + "email_cc_addresses": [ + "test@example.com" + ], + "notified_users": [ + "approver1", + "approver2" + ], "reason": "really important reason!" } }, "response": { "type": "ApiCronJobApproval", "value": { - "approvers": [{ "type": "unicode", "value": "api_test_user" }], + "approvers": [ + { + "type": "unicode", + "value": "api_test_user" + } + ], "email_cc_addresses": [ - { "type": "unicode", "value": "test@example.com" } + { + "type": "unicode", + "value": "test@example.com" + } ], "email_message_id": { "type": "unicode", "value": "" }, - "id": { "type": "unicode", "value": "approval:112233" }, - "is_valid": { "type": "bool", "value": false }, + "id": { + "type": "unicode", + "value": "approval:112233" + }, + "is_valid": { + "type": "bool", + "value": false + }, "is_valid_message": { "type": "unicode", "value": "Need at least 1 additional approver for access." }, "notified_users": [ - { "type": "unicode", "value": "approver1" }, - { "type": "unicode", "value": "approver2" } + { + "type": "unicode", + "value": "approver1" + }, + { + "type": "unicode", + "value": "approver2" + } ], - "reason": { "type": "unicode", "value": "really important reason!" }, - "requestor": { "type": "unicode", "value": "api_test_user" }, + "reason": { + "type": "unicode", + "value": "really important reason!" + }, + "requestor": { + "type": "unicode", + "value": "api_test_user" + }, "subject": { "type": "ApiCronJob", "value": { - "allow_overruns": { "type": "bool", "value": false }, + "allow_overruns": { + "type": "bool", + "value": false + }, "args": { "type": "CronJobAction", "value": { @@ -357,7 +542,10 @@ "hunt_cron_action": { "type": "HuntCronAction", "value": { - "flow_args": { "type": "InterrogateArgs", "value": {} }, + "flow_args": { + "type": "InterrogateArgs", + "value": {} + }, "flow_name": { "type": "unicode", "value": "Interrogate" @@ -377,8 +565,14 @@ "type": "long", "value": 1000 }, - "client_rate": { "type": "float", "value": 20.0 }, - "crash_limit": { "type": "long", "value": 100 } + "client_rate": { + "type": "float", + "value": 20.0 + }, + "crash_limit": { + "type": "long", + "value": 100 + } } } } @@ -389,23 +583,42 @@ "type": "ApiCronJobId", "value": "CronJob_123456" }, - "description": { "type": "unicode", "value": "" }, - "enabled": { "type": "bool", "value": true }, - "frequency": { "type": "DurationSeconds", "value": 86400 }, - "is_failing": { "type": "bool", "value": false } + "description": { + "type": "unicode", + "value": "" + }, + "enabled": { + "type": "bool", + "value": true + }, + "frequency": { + "type": "DurationSeconds", + "value": 86400 + }, + "is_failing": { + "type": "bool", + "value": false + } } } } }, "test_class": "ApiCreateCronJobApprovalHandlerRegressionTest_http_v1", "type_stripped_response": { - "approvers": ["api_test_user"], - "email_cc_addresses": ["test@example.com"], + "approvers": [ + "api_test_user" + ], + "email_cc_addresses": [ + "test@example.com" + ], "email_message_id": "", "id": "approval:112233", "is_valid": false, "is_valid_message": "Need at least 1 additional approver for access.", - "notified_users": ["approver1", "approver2"], + "notified_users": [ + "approver1", + "approver2" + ], "reason": "really important reason!", "requestor": "api_test_user", "subject": { @@ -440,14 +653,26 @@ "method": "POST", "request_payload": { "description": "Foobar!", - "flow_args": { "paths": ["c:\\windows\\system32\\notepad.*"] }, + "flow_args": { + "paths": [ + "c:\\windows\\system32\\notepad.*" + ] + }, "flow_name": "FileFinder", "hunt_runner_args": { "avg_cpu_seconds_per_client_limit": 60, "avg_network_bytes_per_client_limit": 10485760, "avg_results_per_client_limit": 1000, "client_rate": 20.0, - "client_rule_set": { "rules": [{ "os": { "os_windows": true } }] }, + "client_rule_set": { + "rules": [ + { + "os": { + "os_windows": true + } + } + ] + }, "crash_limit": 100, "description": "Foobar! (cron)" }, @@ -457,7 +682,10 @@ "response": { "type": "ApiCronJob", "value": { - "allow_overruns": { "type": "bool", "value": false }, + "allow_overruns": { + "type": "bool", + "value": false + }, "args": { "type": "CronJobAction", "value": { @@ -479,7 +707,10 @@ ] } }, - "flow_name": { "type": "unicode", "value": "FileFinder" }, + "flow_name": { + "type": "unicode", + "value": "FileFinder" + }, "hunt_runner_args": { "type": "HuntRunnerArgs", "value": { @@ -495,7 +726,10 @@ "type": "long", "value": 1000 }, - "client_rate": { "type": "float", "value": 20.0 }, + "client_rate": { + "type": "float", + "value": 20.0 + }, "client_rule_set": { "type": "ForemanClientRuleSet", "value": { @@ -517,7 +751,10 @@ ] } }, - "crash_limit": { "type": "long", "value": 100 }, + "crash_limit": { + "type": "long", + "value": 100 + }, "description": { "type": "unicode", "value": "Foobar! (cron)" @@ -532,11 +769,26 @@ "type": "ApiCronJobId", "value": "CreateAndRunGenericHuntFlow_1234" }, - "description": { "type": "unicode", "value": "Foobar!" }, - "enabled": { "type": "bool", "value": false }, - "frequency": { "type": "DurationSeconds", "value": 604800 }, - "is_failing": { "type": "bool", "value": false }, - "lifetime": { "type": "DurationSeconds", "value": 3600 } + "description": { + "type": "unicode", + "value": "Foobar!" + }, + "enabled": { + "type": "bool", + "value": false + }, + "frequency": { + "type": "DurationSeconds", + "value": 604800 + }, + "is_failing": { + "type": "bool", + "value": false + }, + "lifetime": { + "type": "DurationSeconds", + "value": 3600 + } } }, "test_class": "ApiCreateCronJobHandlerRegressionTest_http_v1", @@ -545,7 +797,11 @@ "args": { "action_type": "HUNT_CRON_ACTION", "hunt_cron_action": { - "flow_args": { "paths": ["c:\\windows\\system32\\notepad.*"] }, + "flow_args": { + "paths": [ + "c:\\windows\\system32\\notepad.*" + ] + }, "flow_name": "FileFinder", "hunt_runner_args": { "avg_cpu_seconds_per_client_limit": 60, @@ -553,7 +809,13 @@ "avg_results_per_client_limit": 1000, "client_rate": 20.0, "client_rule_set": { - "rules": [{ "os": { "os_windows": true } }] + "rules": [ + { + "os": { + "os_windows": true + } + } + ] }, "crash_limit": 100, "description": "Foobar! (cron)" @@ -576,9 +838,14 @@ "method": "POST", "request_payload": { "flow": { - "args": { "fetch_binaries": true, "filename_regex": "." }, + "args": { + "fetch_binaries": true, + "filename_regex": "." + }, "name": "ListProcesses", - "runner_args": { "notify_to_user": true } + "runner_args": { + "notify_to_user": true + } } }, "response": { @@ -587,16 +854,40 @@ "args": { "type": "ListProcessesArgs", "value": { - "fetch_binaries": { "type": "bool", "value": true }, - "filename_regex": { "type": "RegularExpression", "value": "." } + "fetch_binaries": { + "type": "bool", + "value": true + }, + "filename_regex": { + "type": "RegularExpression", + "value": "." + } } }, - "client_id": { "type": "ApiClientId", "value": "C.1000000000000000" }, - "creator": { "type": "unicode", "value": "api_test_user" }, - "flow_id": { "type": "ApiFlowId", "value": "W:ABCDEF" }, - "is_robot": { "type": "bool", "value": false }, - "last_active_at": { "type": "RDFDatetime", "value": 42000000 }, - "name": { "type": "unicode", "value": "ListProcesses" }, + "client_id": { + "type": "ApiClientId", + "value": "C.1000000000000000" + }, + "creator": { + "type": "unicode", + "value": "api_test_user" + }, + "flow_id": { + "type": "ApiFlowId", + "value": "W:ABCDEF" + }, + "is_robot": { + "type": "bool", + "value": false + }, + "last_active_at": { + "type": "RDFDatetime", + "value": 42000000 + }, + "name": { + "type": "unicode", + "value": "ListProcesses" + }, "runner_args": { "type": "FlowRunnerArgs", "value": { @@ -604,12 +895,24 @@ "type": "ClientURN", "value": "aff4:/C.1000000000000000" }, - "flow_name": { "type": "unicode", "value": "ListProcesses" }, - "notify_to_user": { "type": "bool", "value": true } + "flow_name": { + "type": "unicode", + "value": "ListProcesses" + }, + "notify_to_user": { + "type": "bool", + "value": true + } } }, - "started_at": { "type": "RDFDatetime", "value": 42000000 }, - "state": { "type": "EnumNamedValue", "value": "RUNNING" }, + "started_at": { + "type": "RDFDatetime", + "value": 42000000 + }, + "state": { + "type": "EnumNamedValue", + "value": "RUNNING" + }, "urn": { "type": "SessionID", "value": "aff4:/C.1000000000000000/flows/W:ABCDEF" @@ -618,7 +921,10 @@ }, "test_class": "ApiCreateFlowHandlerRegressionTest_http_v1", "type_stripped_response": { - "args": { "fetch_binaries": true, "filename_regex": "." }, + "args": { + "fetch_binaries": true, + "filename_regex": "." + }, "client_id": "C.1000000000000000", "creator": "api_test_user", "flow_id": "W:ABCDEF", @@ -643,17 +949,30 @@ "method": "POST", "request_payload": { "approval": { - "email_cc_addresses": ["test@example.com"], - "notified_users": ["approver1", "approver2"], + "email_cc_addresses": [ + "test@example.com" + ], + "notified_users": [ + "approver1", + "approver2" + ], "reason": "really important reason!" } }, "response": { "type": "ApiHuntApproval", "value": { - "approvers": [{ "type": "unicode", "value": "api_test_user" }], + "approvers": [ + { + "type": "unicode", + "value": "api_test_user" + } + ], "email_cc_addresses": [ - { "type": "unicode", "value": "test@example.com" } + { + "type": "unicode", + "value": "test@example.com" + } ], "email_message_id": { "type": "unicode", @@ -663,24 +982,51 @@ "type": "RDFDatetime", "value": 2419326000000 }, - "id": { "type": "unicode", "value": "approval:112233" }, - "is_valid": { "type": "bool", "value": false }, + "id": { + "type": "unicode", + "value": "approval:112233" + }, + "is_valid": { + "type": "bool", + "value": false + }, "is_valid_message": { "type": "unicode", "value": "Need at least 1 additional approver for access." }, "notified_users": [ - { "type": "unicode", "value": "approver1" }, - { "type": "unicode", "value": "approver2" } + { + "type": "unicode", + "value": "approver1" + }, + { + "type": "unicode", + "value": "approver2" + } ], - "reason": { "type": "unicode", "value": "really important reason!" }, - "requestor": { "type": "unicode", "value": "api_test_user" }, + "reason": { + "type": "unicode", + "value": "really important reason!" + }, + "requestor": { + "type": "unicode", + "value": "api_test_user" + }, "subject": { "type": "ApiHunt", "value": { - "all_clients_count": { "type": "long", "value": 0 }, - "client_limit": { "type": "long", "value": 100 }, - "client_rate": { "type": "float", "value": 0.0 }, + "all_clients_count": { + "type": "long", + "value": 0 + }, + "client_limit": { + "type": "long", + "value": 100 + }, + "client_rate": { + "type": "float", + "value": 0.0 + }, "client_rule_set": { "type": "ForemanClientRuleSet", "value": { @@ -710,32 +1056,71 @@ ] } }, - "clients_with_results_count": { "type": "long", "value": 0 }, - "completed_clients_count": { "type": "long", "value": 0 }, - "crash_limit": { "type": "long", "value": 100 }, - "crashed_clients_count": { "type": "long", "value": 0 }, - "created": { "type": "RDFDatetime", "value": 42000000 }, - "creator": { "type": "unicode", "value": "api_test_user" }, - "description": { "type": "unicode", "value": "foo" }, - "duration": { "type": "DurationSeconds", "value": 1209600 }, - "failed_clients_count": { "type": "long", "value": 0 }, - "flow_args": { - "type": "GetFileArgs", - "value": { - "pathspec": { - "type": "PathSpec", - "value": { - "path": { "type": "unicode", "value": "/tmp/evil.txt" }, - "pathtype": { "type": "EnumNamedValue", "value": "OS" } - } - } - } + "clients_with_results_count": { + "type": "long", + "value": 0 }, - "flow_name": { "type": "unicode", "value": "GetFile" }, - "hunt_id": { "type": "ApiHuntId", "value": "H:123456" }, - "hunt_runner_args": { - "type": "HuntRunnerArgs", - "value": { + "completed_clients_count": { + "type": "long", + "value": 0 + }, + "crash_limit": { + "type": "long", + "value": 100 + }, + "crashed_clients_count": { + "type": "long", + "value": 0 + }, + "created": { + "type": "RDFDatetime", + "value": 42000000 + }, + "creator": { + "type": "unicode", + "value": "api_test_user" + }, + "description": { + "type": "unicode", + "value": "foo" + }, + "duration": { + "type": "DurationSeconds", + "value": 1209600 + }, + "failed_clients_count": { + "type": "long", + "value": 0 + }, + "flow_args": { + "type": "GetFileArgs", + "value": { + "pathspec": { + "type": "PathSpec", + "value": { + "path": { + "type": "unicode", + "value": "/tmp/evil.txt" + }, + "pathtype": { + "type": "EnumNamedValue", + "value": "OS" + } + } + } + } + }, + "flow_name": { + "type": "unicode", + "value": "GetFile" + }, + "hunt_id": { + "type": "ApiHuntId", + "value": "H:123456" + }, + "hunt_runner_args": { + "type": "HuntRunnerArgs", + "value": { "avg_cpu_seconds_per_client_limit": { "type": "long", "value": 60 @@ -748,7 +1133,10 @@ "type": "long", "value": 1000 }, - "client_rate": { "type": "float", "value": 0.0 }, + "client_rate": { + "type": "float", + "value": 0.0 + }, "client_rule_set": { "type": "ForemanClientRuleSet", "value": { @@ -778,43 +1166,93 @@ ] } }, - "crash_limit": { "type": "long", "value": 100 }, - "description": { "type": "unicode", "value": "foo" }, + "crash_limit": { + "type": "long", + "value": 100 + }, + "description": { + "type": "unicode", + "value": "foo" + }, "expiry_time": { "type": "DurationSeconds", "value": 1209600 }, - "hunt_name": { "type": "unicode", "value": "GenericHunt" }, + "hunt_name": { + "type": "unicode", + "value": "GenericHunt" + }, "original_object": { "type": "FlowLikeObjectReference", "value": {} } } }, - "hunt_type": { "type": "EnumNamedValue", "value": "STANDARD" }, - "is_robot": { "type": "bool", "value": false }, - "name": { "type": "unicode", "value": "GenericHunt" }, - "remaining_clients_count": { "type": "long", "value": 0 }, - "results_count": { "type": "long", "value": 0 }, - "state": { "type": "EnumNamedValue", "value": "PAUSED" }, - "state_comment": { "type": "unicode", "value": "" }, - "total_cpu_usage": { "type": "long", "value": 0 }, - "total_net_usage": { "type": "long", "value": 0 }, - "urn": { "type": "SessionID", "value": "aff4:/hunts/H:123456" } + "hunt_type": { + "type": "EnumNamedValue", + "value": "STANDARD" + }, + "is_robot": { + "type": "bool", + "value": false + }, + "name": { + "type": "unicode", + "value": "GenericHunt" + }, + "remaining_clients_count": { + "type": "long", + "value": 0 + }, + "results_count": { + "type": "long", + "value": 0 + }, + "state": { + "type": "EnumNamedValue", + "value": "PAUSED" + }, + "state_comment": { + "type": "unicode", + "value": "" + }, + "state_reason": { + "type": "EnumNamedValue", + "value": "UNKNOWN" + }, + "total_cpu_usage": { + "type": "long", + "value": 0 + }, + "total_net_usage": { + "type": "long", + "value": 0 + }, + "urn": { + "type": "SessionID", + "value": "aff4:/hunts/H:123456" + } } } } }, "test_class": "ApiCreateHuntApprovalHandlerRegressionTest_http_v1", "type_stripped_response": { - "approvers": ["api_test_user"], - "email_cc_addresses": ["test@example.com"], + "approvers": [ + "api_test_user" + ], + "email_cc_addresses": [ + "test@example.com" + ], "email_message_id": "", "expiration_time_us": 2419326000000, "id": "approval:112233", "is_valid": false, "is_valid_message": "Need at least 1 additional approver for access.", - "notified_users": ["approver1", "approver2"], + "notified_users": [ + "approver1", + "approver2" + ], "reason": "really important reason!", "requestor": "api_test_user", "subject": { @@ -824,7 +1262,10 @@ "client_rule_set": { "rules": [ { - "regex": { "attribute_regex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attribute_regex": "GRR", + "field": "CLIENT_NAME" + }, "rule_type": "REGEX" } ] @@ -839,7 +1280,10 @@ "duration": 1209600, "failed_clients_count": 0, "flow_args": { - "pathspec": { "path": "/tmp/evil.txt", "pathtype": "OS" } + "pathspec": { + "path": "/tmp/evil.txt", + "pathtype": "OS" + } }, "flow_name": "GetFile", "hunt_id": "H:123456", @@ -851,7 +1295,10 @@ "client_rule_set": { "rules": [ { - "regex": { "attribute_regex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attribute_regex": "GRR", + "field": "CLIENT_NAME" + }, "rule_type": "REGEX" } ] @@ -869,6 +1316,7 @@ "results_count": 0, "state": "PAUSED", "state_comment": "", + "state_reason": "UNKNOWN", "total_cpu_usage": 0, "total_net_usage": 0, "urn": "aff4:/hunts/H:123456" @@ -886,28 +1334,64 @@ "per_client_args": [ { "client_id": "C.1000000000000000", - "paths": ["/etc/hosts", "/foo/bar"] + "paths": [ + "/etc/hosts", + "/foo/bar" + ] } ] }, "response": { "type": "ApiHunt", "value": { - "all_clients_count": { "type": "long", "value": 0 }, - "client_limit": { "type": "long", "value": 100 }, - "client_rate": { "type": "long", "value": 0 }, - "client_rule_set": { "type": "ForemanClientRuleSet", "value": {} }, - "clients_with_results_count": { "type": "long", "value": 0 }, - "completed_clients_count": { "type": "long", "value": 0 }, - "crash_limit": { "type": "long", "value": 100 }, - "crashed_clients_count": { "type": "long", "value": 0 }, - "creator": { "type": "unicode", "value": "api_test_user" }, + "all_clients_count": { + "type": "long", + "value": 0 + }, + "client_limit": { + "type": "long", + "value": 100 + }, + "client_rate": { + "type": "long", + "value": 0 + }, + "client_rule_set": { + "type": "ForemanClientRuleSet", + "value": {} + }, + "clients_with_results_count": { + "type": "long", + "value": 0 + }, + "completed_clients_count": { + "type": "long", + "value": 0 + }, + "crash_limit": { + "type": "long", + "value": 100 + }, + "crashed_clients_count": { + "type": "long", + "value": 0 + }, + "creator": { + "type": "unicode", + "value": "api_test_user" + }, "description": { "type": "unicode", "value": "Per-client file collection" }, - "duration": { "type": "DurationSeconds", "value": 1209600 }, - "failed_clients_count": { "type": "long", "value": 0 }, + "duration": { + "type": "DurationSeconds", + "value": 1209600 + }, + "failed_clients_count": { + "type": "long", + "value": 0 + }, "flow_args": { "type": "HuntArgumentsVariable", "value": { @@ -916,7 +1400,10 @@ "type": "VariableHuntFlowGroup", "value": { "client_ids": [ - { "type": "unicode", "value": "C.1000000000000000" } + { + "type": "unicode", + "value": "C.1000000000000000" + } ], "flow_args": { "type": "MultiGetFileArgs", @@ -951,13 +1438,19 @@ ] } }, - "flow_name": { "type": "unicode", "value": "MultiGetFile" } + "flow_name": { + "type": "unicode", + "value": "MultiGetFile" + } } } ] } }, - "hunt_id": { "type": "ApiHuntId", "value": "H:123456" }, + "hunt_id": { + "type": "ApiHuntId", + "value": "H:123456" + }, "hunt_runner_args": { "type": "HuntRunnerArgs", "value": { @@ -969,18 +1462,30 @@ "type": "long", "value": 10485760 }, - "avg_results_per_client_limit": { "type": "long", "value": 1000 }, - "client_rate": { "type": "long", "value": 0 }, + "avg_results_per_client_limit": { + "type": "long", + "value": 1000 + }, + "client_rate": { + "type": "long", + "value": 0 + }, "client_rule_set": { "type": "ForemanClientRuleSet", "value": {} }, - "crash_limit": { "type": "long", "value": 100 }, + "crash_limit": { + "type": "long", + "value": 100 + }, "description": { "type": "unicode", "value": "Per-client file collection" }, - "expiry_time": { "type": "DurationSeconds", "value": 1209600 }, + "expiry_time": { + "type": "DurationSeconds", + "value": 1209600 + }, "hunt_name": { "type": "unicode", "value": "VariableGenericHunt" @@ -991,16 +1496,50 @@ } } }, - "hunt_type": { "type": "EnumNamedValue", "value": "VARIABLE" }, - "is_robot": { "type": "bool", "value": false }, - "name": { "type": "unicode", "value": "VariableGenericHunt" }, - "remaining_clients_count": { "type": "long", "value": 0 }, - "results_count": { "type": "long", "value": 0 }, - "state": { "type": "EnumNamedValue", "value": "PAUSED" }, - "state_comment": { "type": "unicode", "value": "" }, - "total_cpu_usage": { "type": "long", "value": 0 }, - "total_net_usage": { "type": "long", "value": 0 }, - "urn": { "type": "SessionID", "value": "aff4:/hunts/H:123456" } + "hunt_type": { + "type": "EnumNamedValue", + "value": "VARIABLE" + }, + "is_robot": { + "type": "bool", + "value": false + }, + "name": { + "type": "unicode", + "value": "VariableGenericHunt" + }, + "remaining_clients_count": { + "type": "long", + "value": 0 + }, + "results_count": { + "type": "long", + "value": 0 + }, + "state": { + "type": "EnumNamedValue", + "value": "PAUSED" + }, + "state_comment": { + "type": "unicode", + "value": "" + }, + "state_reason": { + "type": "EnumNamedValue", + "value": "UNKNOWN" + }, + "total_cpu_usage": { + "type": "long", + "value": 0 + }, + "total_net_usage": { + "type": "long", + "value": 0 + }, + "urn": { + "type": "SessionID", + "value": "aff4:/hunts/H:123456" + } } }, "test_class": "ApiCreatePerClientFileCollectionHuntRegressionTest_http_v1", @@ -1020,11 +1559,19 @@ "flow_args": { "flow_groups": [ { - "client_ids": ["C.1000000000000000"], + "client_ids": [ + "C.1000000000000000" + ], "flow_args": { "pathspecs": [ - { "path": "/etc/hosts", "pathtype": "OS" }, - { "path": "/foo/bar", "pathtype": "OS" } + { + "path": "/etc/hosts", + "pathtype": "OS" + }, + { + "path": "/foo/bar", + "pathtype": "OS" + } ] }, "flow_name": "MultiGetFile" @@ -1051,6 +1598,7 @@ "results_count": 0, "state": "PAUSED", "state_comment": "", + "state_reason": "UNKNOWN", "total_cpu_usage": 0, "total_net_usage": 0, "urn": "aff4:/hunts/H:123456" @@ -1062,8 +1610,13 @@ { "api_method": "CreateVfsRefreshOperation", "method": "POST", - "request_payload": { "file_path": "fs/os/Users/Shared", "max_depth": 1 }, - "response": { "operation_id": "ABCDEF" }, + "request_payload": { + "file_path": "fs/os/Users/Shared", + "max_depth": 1 + }, + "response": { + "operation_id": "ABCDEF" + }, "test_class": "ApiCreateVfsRefreshOperationHandlerRegressionTest_http_v1", "url": "/api/clients/C.1000000000000000/vfs-refresh-operations" } @@ -1072,7 +1625,9 @@ { "api_method": "DeleteHunt", "method": "DELETE", - "response": { "status": "OK" }, + "response": { + "status": "OK" + }, "test_class": "ApiDeleteHuntHandlerRegressionTest_http_v1", "url": "/api/hunts/H:123456" } @@ -1081,21 +1636,29 @@ { "api_method": "ExplainGlobExpression", "method": "POST", - "request_payload": { "glob_expression": "/foo/*" }, + "request_payload": { + "glob_expression": "/foo/*" + }, "response": { "components": [ { "type": "GlobComponentExplanation", "value": { "examples": [], - "glob_expression": { "type": "unicode", "value": "/foo/" } + "glob_expression": { + "type": "unicode", + "value": "/foo/" + } } }, { "type": "GlobComponentExplanation", "value": { "examples": [], - "glob_expression": { "type": "unicode", "value": "*" } + "glob_expression": { + "type": "unicode", + "value": "*" + } } } ] @@ -1103,8 +1666,14 @@ "test_class": "ApiExplainGlobExpressionHandlerTest_http_v1", "type_stripped_response": { "components": [ - { "examples": [], "glob_expression": "/foo/" }, - { "examples": [], "glob_expression": "*" } + { + "examples": [], + "glob_expression": "/foo/" + }, + { + "examples": [], + "glob_expression": "*" + } ] }, "url": "/api/clients/C.1000000000000000/glob-expressions:explain" @@ -1114,7 +1683,9 @@ { "api_method": "ForceRunCronJob", "method": "POST", - "response": { "status": "OK" }, + "response": { + "status": "OK" + }, "test_class": "ApiForceRunCronJobRegressionTest_http_v1", "url": "/api/cron-jobs/FileFinder/actions/force-run" } @@ -1126,7 +1697,12 @@ "response": { "type": "ApiClientApproval", "value": { - "approvers": [{ "type": "unicode", "value": "api_test_user" }], + "approvers": [ + { + "type": "unicode", + "value": "api_test_user" + } + ], "email_cc_addresses": [], "email_message_id": { "type": "unicode", @@ -1136,28 +1712,63 @@ "type": "RDFDatetime", "value": 2419244000000 }, - "id": { "type": "unicode", "value": "approval:111111" }, - "is_valid": { "type": "bool", "value": false }, + "id": { + "type": "unicode", + "value": "approval:111111" + }, + "is_valid": { + "type": "bool", + "value": false + }, "is_valid_message": { "type": "unicode", "value": "Need at least 1 additional approver for access." }, - "notified_users": [{ "type": "unicode", "value": "approver" }], - "reason": { "type": "unicode", "value": "foo" }, - "requestor": { "type": "unicode", "value": "api_test_user" }, + "notified_users": [ + { + "type": "unicode", + "value": "approver" + } + ], + "reason": { + "type": "unicode", + "value": "foo" + }, + "requestor": { + "type": "unicode", + "value": "api_test_user" + }, "subject": { "type": "ApiClient", "value": { - "age": { "type": "RDFDatetime", "value": 42000000 }, + "age": { + "type": "RDFDatetime", + "value": 42000000 + }, "agent_info": { "type": "ClientInformation", "value": { - "build_time": { "type": "unicode", "value": "1980-01-01" }, - "client_name": { "type": "unicode", "value": "GRR Monitor" }, - "client_version": { "type": "long", "value": 1234 }, + "build_time": { + "type": "unicode", + "value": "1980-01-01" + }, + "client_name": { + "type": "unicode", + "value": "GRR Monitor" + }, + "client_version": { + "type": "long", + "value": 1234 + }, "labels": [ - { "type": "unicode", "value": "label1" }, - { "type": "unicode", "value": "label2" } + { + "type": "unicode", + "value": "label1" + }, + { + "type": "unicode", + "value": "label2" + } ] } }, @@ -1165,7 +1776,6 @@ "type": "ApiClientId", "value": "C.1000000000000000" }, - "fleetspeak_enabled": { "type": "bool", "value": false }, "hardware_info": { "type": "HardwareInfo", "value": { @@ -1211,55 +1821,100 @@ } } ], - "ifname": { "type": "unicode", "value": "if0" } + "ifname": { + "type": "unicode", + "value": "if0" + } } }, { "type": "Interface", "value": { - "ifname": { "type": "unicode", "value": "if1" }, - "mac_address": { "type": "MacAddress", "value": "qrvM3e4A" } + "ifname": { + "type": "unicode", + "value": "if1" + }, + "mac_address": { + "type": "MacAddress", + "value": "qrvM3e4A" + } } }, { "type": "Interface", "value": { - "ifname": { "type": "unicode", "value": "if2" }, - "mac_address": { "type": "MacAddress", "value": "u8zd7v8A" } + "ifname": { + "type": "unicode", + "value": "if2" + }, + "mac_address": { + "type": "MacAddress", + "value": "u8zd7v8A" + } } } ], "knowledge_base": { "type": "KnowledgeBase", "value": { - "fqdn": { "type": "unicode", "value": "Host-0.example.com" }, - "os": { "type": "unicode", "value": "Linux" }, + "fqdn": { + "type": "unicode", + "value": "Host-0.example.com" + }, + "os": { + "type": "unicode", + "value": "Linux" + }, "users": [ { "type": "User", "value": { - "username": { "type": "unicode", "value": "user1" } + "username": { + "type": "unicode", + "value": "user1" + } } }, { "type": "User", "value": { - "username": { "type": "unicode", "value": "user2" } + "username": { + "type": "unicode", + "value": "user2" + } } } ] } }, "labels": [], - "last_seen_at": { "type": "RDFDatetime", "value": 42000000 }, + "last_seen_at": { + "type": "RDFDatetime", + "value": 42000000 + }, "os_info": { "type": "Uname", "value": { - "fqdn": { "type": "unicode", "value": "Host-0.example.com" }, - "kernel": { "type": "unicode", "value": "4.0.0" }, - "machine": { "type": "unicode", "value": "x86_64" }, - "system": { "type": "unicode", "value": "Linux" }, - "version": { "type": "unicode", "value": "buster/sid" } + "fqdn": { + "type": "unicode", + "value": "Host-0.example.com" + }, + "kernel": { + "type": "unicode", + "value": "4.0.0" + }, + "machine": { + "type": "unicode", + "value": "x86_64" + }, + "system": { + "type": "unicode", + "value": "Linux" + }, + "version": { + "type": "unicode", + "value": "buster/sid" + } } }, "urn": { @@ -1270,13 +1925,19 @@ { "type": "User", "value": { - "username": { "type": "unicode", "value": "user1" } + "username": { + "type": "unicode", + "value": "user1" + } } }, { "type": "User", "value": { - "username": { "type": "unicode", "value": "user2" } + "username": { + "type": "unicode", + "value": "user2" + } } } ] @@ -1286,14 +1947,18 @@ }, "test_class": "ApiGetClientApprovalHandlerRegressionTest_http_v1", "type_stripped_response": { - "approvers": ["api_test_user"], + "approvers": [ + "api_test_user" + ], "email_cc_addresses": [], "email_message_id": "", "expiration_time_us": 2419244000000, "id": "approval:111111", "is_valid": false, "is_valid_message": "Need at least 1 additional approver for access.", - "notified_users": ["approver"], + "notified_users": [ + "approver" + ], "reason": "foo", "requestor": "api_test_user", "subject": { @@ -1302,10 +1967,12 @@ "build_time": "1980-01-01", "client_name": "GRR Monitor", "client_version": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "client_id": "C.1000000000000000", - "fleetspeak_enabled": false, "hardware_info": { "bios_version": "Bios-Version-0", "system_manufacturer": "System-Manufacturer-0" @@ -1313,7 +1980,10 @@ "interfaces": [ { "addresses": [ - { "address_type": "INET", "packed_bytes": "wKgAAA==" }, + { + "address_type": "INET", + "packed_bytes": "wKgAAA==" + }, { "address_type": "INET6", "packed_bytes": "IAGrzQAAAAAAAAAAAAAAAA==" @@ -1321,13 +1991,26 @@ ], "ifname": "if0" }, - { "ifname": "if1", "mac_address": "qrvM3e4A" }, - { "ifname": "if2", "mac_address": "u8zd7v8A" } + { + "ifname": "if1", + "mac_address": "qrvM3e4A" + }, + { + "ifname": "if2", + "mac_address": "u8zd7v8A" + } ], "knowledge_base": { "fqdn": "Host-0.example.com", "os": "Linux", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] }, "labels": [], "last_seen_at": 42000000, @@ -1339,7 +2022,14 @@ "version": "buster/sid" }, "urn": "aff4:/C.1000000000000000", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] } }, "url": "/api/users/api_test_user/approvals/client/C.1000000000000000/approval:111111" @@ -1351,8 +2041,14 @@ "type": "ApiClientApproval", "value": { "approvers": [ - { "type": "unicode", "value": "api_test_user" }, - { "type": "unicode", "value": "approver" } + { + "type": "unicode", + "value": "api_test_user" + }, + { + "type": "unicode", + "value": "approver" + } ], "email_cc_addresses": [], "email_message_id": { @@ -1363,32 +2059,66 @@ "type": "RDFDatetime", "value": 2419245000000 }, - "id": { "type": "unicode", "value": "approval:222222" }, - "is_valid": { "type": "bool", "value": true }, - "notified_users": [{ "type": "unicode", "value": "approver" }], - "reason": { "type": "unicode", "value": "bar" }, - "requestor": { "type": "unicode", "value": "api_test_user" }, + "id": { + "type": "unicode", + "value": "approval:222222" + }, + "is_valid": { + "type": "bool", + "value": true + }, + "notified_users": [ + { + "type": "unicode", + "value": "approver" + } + ], + "reason": { + "type": "unicode", + "value": "bar" + }, + "requestor": { + "type": "unicode", + "value": "api_test_user" + }, "subject": { "type": "ApiClient", "value": { - "age": { "type": "RDFDatetime", "value": 42000000 }, + "age": { + "type": "RDFDatetime", + "value": 42000000 + }, "agent_info": { "type": "ClientInformation", "value": { - "build_time": { "type": "unicode", "value": "1980-01-01" }, - "client_name": { "type": "unicode", "value": "GRR Monitor" }, - "client_version": { "type": "long", "value": 1234 }, + "build_time": { + "type": "unicode", + "value": "1980-01-01" + }, + "client_name": { + "type": "unicode", + "value": "GRR Monitor" + }, + "client_version": { + "type": "long", + "value": 1234 + }, "labels": [ - { "type": "unicode", "value": "label1" }, - { "type": "unicode", "value": "label2" } - ] + { + "type": "unicode", + "value": "label1" + }, + { + "type": "unicode", + "value": "label2" + } + ] } }, "client_id": { "type": "ApiClientId", "value": "C.1000000000000001" }, - "fleetspeak_enabled": { "type": "bool", "value": false }, "hardware_info": { "type": "HardwareInfo", "value": { @@ -1434,55 +2164,100 @@ } } ], - "ifname": { "type": "unicode", "value": "if0" } + "ifname": { + "type": "unicode", + "value": "if0" + } } }, { "type": "Interface", "value": { - "ifname": { "type": "unicode", "value": "if1" }, - "mac_address": { "type": "MacAddress", "value": "qrvM3e4B" } + "ifname": { + "type": "unicode", + "value": "if1" + }, + "mac_address": { + "type": "MacAddress", + "value": "qrvM3e4B" + } } }, { "type": "Interface", "value": { - "ifname": { "type": "unicode", "value": "if2" }, - "mac_address": { "type": "MacAddress", "value": "u8zd7v8B" } + "ifname": { + "type": "unicode", + "value": "if2" + }, + "mac_address": { + "type": "MacAddress", + "value": "u8zd7v8B" + } } } ], "knowledge_base": { "type": "KnowledgeBase", "value": { - "fqdn": { "type": "unicode", "value": "Host-1.example.com" }, - "os": { "type": "unicode", "value": "Linux" }, + "fqdn": { + "type": "unicode", + "value": "Host-1.example.com" + }, + "os": { + "type": "unicode", + "value": "Linux" + }, "users": [ { "type": "User", "value": { - "username": { "type": "unicode", "value": "user1" } + "username": { + "type": "unicode", + "value": "user1" + } } }, { "type": "User", "value": { - "username": { "type": "unicode", "value": "user2" } + "username": { + "type": "unicode", + "value": "user2" + } } } ] } }, "labels": [], - "last_seen_at": { "type": "RDFDatetime", "value": 42000000 }, + "last_seen_at": { + "type": "RDFDatetime", + "value": 42000000 + }, "os_info": { "type": "Uname", "value": { - "fqdn": { "type": "unicode", "value": "Host-1.example.com" }, - "kernel": { "type": "unicode", "value": "4.0.0" }, - "machine": { "type": "unicode", "value": "x86_64" }, - "system": { "type": "unicode", "value": "Linux" }, - "version": { "type": "unicode", "value": "buster/sid" } + "fqdn": { + "type": "unicode", + "value": "Host-1.example.com" + }, + "kernel": { + "type": "unicode", + "value": "4.0.0" + }, + "machine": { + "type": "unicode", + "value": "x86_64" + }, + "system": { + "type": "unicode", + "value": "Linux" + }, + "version": { + "type": "unicode", + "value": "buster/sid" + } } }, "urn": { @@ -1493,13 +2268,19 @@ { "type": "User", "value": { - "username": { "type": "unicode", "value": "user1" } + "username": { + "type": "unicode", + "value": "user1" + } } }, { "type": "User", "value": { - "username": { "type": "unicode", "value": "user2" } + "username": { + "type": "unicode", + "value": "user2" + } } } ] @@ -1509,13 +2290,18 @@ }, "test_class": "ApiGetClientApprovalHandlerRegressionTest_http_v1", "type_stripped_response": { - "approvers": ["api_test_user", "approver"], + "approvers": [ + "api_test_user", + "approver" + ], "email_cc_addresses": [], "email_message_id": "", "expiration_time_us": 2419245000000, "id": "approval:222222", "is_valid": true, - "notified_users": ["approver"], + "notified_users": [ + "approver" + ], "reason": "bar", "requestor": "api_test_user", "subject": { @@ -1524,10 +2310,12 @@ "build_time": "1980-01-01", "client_name": "GRR Monitor", "client_version": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "client_id": "C.1000000000000001", - "fleetspeak_enabled": false, "hardware_info": { "bios_version": "Bios-Version-1", "system_manufacturer": "System-Manufacturer-1" @@ -1535,7 +2323,10 @@ "interfaces": [ { "addresses": [ - { "address_type": "INET", "packed_bytes": "wKgAAQ==" }, + { + "address_type": "INET", + "packed_bytes": "wKgAAQ==" + }, { "address_type": "INET6", "packed_bytes": "IAGrzQAAAAAAAAAAAAAAAQ==" @@ -1543,13 +2334,26 @@ ], "ifname": "if0" }, - { "ifname": "if1", "mac_address": "qrvM3e4B" }, - { "ifname": "if2", "mac_address": "u8zd7v8B" } + { + "ifname": "if1", + "mac_address": "qrvM3e4B" + }, + { + "ifname": "if2", + "mac_address": "u8zd7v8B" + } ], "knowledge_base": { "fqdn": "Host-1.example.com", "os": "Linux", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] }, "labels": [], "last_seen_at": 42000000, @@ -1561,7 +2365,14 @@ "version": "buster/sid" }, "urn": "aff4:/C.1000000000000001", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] } }, "url": "/api/users/api_test_user/approvals/client/C.1000000000000001/approval:222222" @@ -1574,25 +2385,48 @@ "response": { "type": "ApiClient", "value": { - "age": { "type": "RDFDatetime", "value": 42000000 }, + "age": { + "type": "RDFDatetime", + "value": 42000000 + }, "agent_info": { "type": "ClientInformation", "value": { - "build_time": { "type": "unicode", "value": "1980-01-01" }, - "client_name": { "type": "unicode", "value": "GRR Monitor" }, - "client_version": { "type": "long", "value": 1234 }, + "build_time": { + "type": "unicode", + "value": "1980-01-01" + }, + "client_name": { + "type": "unicode", + "value": "GRR Monitor" + }, + "client_version": { + "type": "long", + "value": 1234 + }, "labels": [ - { "type": "unicode", "value": "label1" }, - { "type": "unicode", "value": "label2" } + { + "type": "unicode", + "value": "label1" + }, + { + "type": "unicode", + "value": "label2" + } ] } }, - "client_id": { "type": "ApiClientId", "value": "C.1000000000000000" }, - "fleetspeak_enabled": { "type": "bool", "value": false }, + "client_id": { + "type": "ApiClientId", + "value": "C.1000000000000000" + }, "hardware_info": { "type": "HardwareInfo", "value": { - "bios_version": { "type": "unicode", "value": "Bios-Version-0" }, + "bios_version": { + "type": "unicode", + "value": "Bios-Version-0" + }, "system_manufacturer": { "type": "unicode", "value": "System-Manufacturer-0" @@ -1611,7 +2445,10 @@ "type": "EnumNamedValue", "value": "INET" }, - "packed_bytes": { "type": "bytes", "value": "wKgAAA==" } + "packed_bytes": { + "type": "bytes", + "value": "wKgAAA==" + } } }, { @@ -1628,67 +2465,128 @@ } } ], - "ifname": { "type": "unicode", "value": "if0" } + "ifname": { + "type": "unicode", + "value": "if0" + } } }, { "type": "Interface", "value": { - "ifname": { "type": "unicode", "value": "if1" }, - "mac_address": { "type": "MacAddress", "value": "qrvM3e4A" } + "ifname": { + "type": "unicode", + "value": "if1" + }, + "mac_address": { + "type": "MacAddress", + "value": "qrvM3e4A" + } } }, { "type": "Interface", "value": { - "ifname": { "type": "unicode", "value": "if2" }, - "mac_address": { "type": "MacAddress", "value": "u8zd7v8A" } + "ifname": { + "type": "unicode", + "value": "if2" + }, + "mac_address": { + "type": "MacAddress", + "value": "u8zd7v8A" + } } } ], "knowledge_base": { "type": "KnowledgeBase", "value": { - "fqdn": { "type": "unicode", "value": "Host-0.example.com" }, - "os": { "type": "unicode", "value": "Linux" }, + "fqdn": { + "type": "unicode", + "value": "Host-0.example.com" + }, + "os": { + "type": "unicode", + "value": "Linux" + }, "users": [ { "type": "User", "value": { - "username": { "type": "unicode", "value": "user1" } + "username": { + "type": "unicode", + "value": "user1" + } } }, { "type": "User", "value": { - "username": { "type": "unicode", "value": "user2" } + "username": { + "type": "unicode", + "value": "user2" + } } } ] } }, "labels": [], - "last_seen_at": { "type": "RDFDatetime", "value": 42000000 }, - "memory_size": { "type": "ByteSize", "value": 4294967296 }, + "last_seen_at": { + "type": "RDFDatetime", + "value": 42000000 + }, + "memory_size": { + "type": "ByteSize", + "value": 4294967296 + }, "os_info": { "type": "Uname", "value": { - "fqdn": { "type": "unicode", "value": "Host-0.example.com" }, - "kernel": { "type": "unicode", "value": "4.0.0" }, - "machine": { "type": "unicode", "value": "x86_64" }, - "system": { "type": "unicode", "value": "Linux" }, - "version": { "type": "unicode", "value": "buster/sid" } + "fqdn": { + "type": "unicode", + "value": "Host-0.example.com" + }, + "kernel": { + "type": "unicode", + "value": "4.0.0" + }, + "machine": { + "type": "unicode", + "value": "x86_64" + }, + "system": { + "type": "unicode", + "value": "Linux" + }, + "version": { + "type": "unicode", + "value": "buster/sid" + } } }, - "urn": { "type": "ClientURN", "value": "aff4:/C.1000000000000000" }, + "urn": { + "type": "ClientURN", + "value": "aff4:/C.1000000000000000" + }, "users": [ { "type": "User", - "value": { "username": { "type": "unicode", "value": "user1" } } + "value": { + "username": { + "type": "unicode", + "value": "user1" + } + } }, { "type": "User", - "value": { "username": { "type": "unicode", "value": "user2" } } + "value": { + "username": { + "type": "unicode", + "value": "user2" + } + } } ] } @@ -1700,10 +2598,12 @@ "build_time": "1980-01-01", "client_name": "GRR Monitor", "client_version": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "client_id": "C.1000000000000000", - "fleetspeak_enabled": false, "hardware_info": { "bios_version": "Bios-Version-0", "system_manufacturer": "System-Manufacturer-0" @@ -1711,7 +2611,10 @@ "interfaces": [ { "addresses": [ - { "address_type": "INET", "packed_bytes": "wKgAAA==" }, + { + "address_type": "INET", + "packed_bytes": "wKgAAA==" + }, { "address_type": "INET6", "packed_bytes": "IAGrzQAAAAAAAAAAAAAAAA==" @@ -1719,13 +2622,26 @@ ], "ifname": "if0" }, - { "ifname": "if1", "mac_address": "qrvM3e4A" }, - { "ifname": "if2", "mac_address": "u8zd7v8A" } + { + "ifname": "if1", + "mac_address": "qrvM3e4A" + }, + { + "ifname": "if2", + "mac_address": "u8zd7v8A" + } ], "knowledge_base": { "fqdn": "Host-0.example.com", "os": "Linux", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] }, "labels": [], "last_seen_at": 42000000, @@ -1738,7 +2654,14 @@ "version": "buster/sid" }, "urn": "aff4:/C.1000000000000000", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] }, "url": "/api/clients/C.1000000000000000" } @@ -1749,8 +2672,14 @@ "method": "GET", "response": { "data_points": [ - [10000.0, 10.0], - [20000.0, 11.0] + [ + 10000.0, + 10.0 + ], + [ + 20000.0, + 11.0 + ] ] }, "test_class": "ApiGetClientLoadStatsHandlerRegressionTest_http_v1", @@ -1761,8 +2690,14 @@ "method": "GET", "response": { "data_points": [ - [10000.0, 10.0], - [20000.0, 12.0] + [ + 10000.0, + 10.0 + ], + [ + 20000.0, + 12.0 + ] ] }, "test_class": "ApiGetClientLoadStatsHandlerRegressionTest_http_v1", @@ -1778,16 +2713,34 @@ { "type": "ApiClient", "value": { - "age": { "type": "RDFDatetime", "value": 42000000 }, + "age": { + "type": "RDFDatetime", + "value": 42000000 + }, "agent_info": { "type": "ClientInformation", "value": { - "build_time": { "type": "unicode", "value": "1980-01-01" }, - "client_name": { "type": "unicode", "value": "GRR Monitor" }, - "client_version": { "type": "long", "value": 1234 }, + "build_time": { + "type": "unicode", + "value": "1980-01-01" + }, + "client_name": { + "type": "unicode", + "value": "GRR Monitor" + }, + "client_version": { + "type": "long", + "value": 1234 + }, "labels": [ - { "type": "unicode", "value": "label1" }, - { "type": "unicode", "value": "label2" } + { + "type": "unicode", + "value": "label1" + }, + { + "type": "unicode", + "value": "label2" + } ] } }, @@ -1840,55 +2793,100 @@ } } ], - "ifname": { "type": "unicode", "value": "if0" } + "ifname": { + "type": "unicode", + "value": "if0" + } } }, { "type": "Interface", "value": { - "ifname": { "type": "unicode", "value": "if1" }, - "mac_address": { "type": "MacAddress", "value": "qrvM3e4A" } + "ifname": { + "type": "unicode", + "value": "if1" + }, + "mac_address": { + "type": "MacAddress", + "value": "qrvM3e4A" + } } }, { "type": "Interface", "value": { - "ifname": { "type": "unicode", "value": "if2" }, - "mac_address": { "type": "MacAddress", "value": "u8zd7v8A" } + "ifname": { + "type": "unicode", + "value": "if2" + }, + "mac_address": { + "type": "MacAddress", + "value": "u8zd7v8A" + } } } ], "knowledge_base": { "type": "KnowledgeBase", "value": { - "fqdn": { "type": "unicode", "value": "Host-0.example.com" }, - "os": { "type": "unicode", "value": "Linux" }, + "fqdn": { + "type": "unicode", + "value": "Host-0.example.com" + }, + "os": { + "type": "unicode", + "value": "Linux" + }, "users": [ { "type": "User", "value": { - "username": { "type": "unicode", "value": "user1" } + "username": { + "type": "unicode", + "value": "user1" + } } }, { "type": "User", "value": { - "username": { "type": "unicode", "value": "user2" } + "username": { + "type": "unicode", + "value": "user2" + } } } ] } }, "labels": [], - "memory_size": { "type": "ByteSize", "value": 4294967296 }, + "memory_size": { + "type": "ByteSize", + "value": 4294967296 + }, "os_info": { "type": "Uname", "value": { - "fqdn": { "type": "unicode", "value": "Host-0.example.com" }, - "kernel": { "type": "unicode", "value": "4.0.0" }, - "machine": { "type": "unicode", "value": "x86_64" }, - "system": { "type": "unicode", "value": "Linux" }, - "version": { "type": "unicode", "value": "buster/sid" } + "fqdn": { + "type": "unicode", + "value": "Host-0.example.com" + }, + "kernel": { + "type": "unicode", + "value": "4.0.0" + }, + "machine": { + "type": "unicode", + "value": "x86_64" + }, + "system": { + "type": "unicode", + "value": "Linux" + }, + "version": { + "type": "unicode", + "value": "buster/sid" + } } }, "urn": { @@ -1899,13 +2897,19 @@ { "type": "User", "value": { - "username": { "type": "unicode", "value": "user1" } + "username": { + "type": "unicode", + "value": "user1" + } } }, { "type": "User", "value": { - "username": { "type": "unicode", "value": "user2" } + "username": { + "type": "unicode", + "value": "user2" + } } } ] @@ -1914,16 +2918,34 @@ { "type": "ApiClient", "value": { - "age": { "type": "RDFDatetime", "value": 45000000 }, + "age": { + "type": "RDFDatetime", + "value": 45000000 + }, "agent_info": { "type": "ClientInformation", "value": { - "build_time": { "type": "unicode", "value": "1980-01-01" }, - "client_name": { "type": "unicode", "value": "GRR Monitor" }, - "client_version": { "type": "long", "value": 1234 }, + "build_time": { + "type": "unicode", + "value": "1980-01-01" + }, + "client_name": { + "type": "unicode", + "value": "GRR Monitor" + }, + "client_version": { + "type": "long", + "value": 1234 + }, "labels": [ - { "type": "unicode", "value": "label1" }, - { "type": "unicode", "value": "label2" } + { + "type": "unicode", + "value": "label1" + }, + { + "type": "unicode", + "value": "label2" + } ] } }, @@ -1976,21 +2998,36 @@ } } ], - "ifname": { "type": "unicode", "value": "if0" } + "ifname": { + "type": "unicode", + "value": "if0" + } } }, { "type": "Interface", "value": { - "ifname": { "type": "unicode", "value": "if1" }, - "mac_address": { "type": "MacAddress", "value": "qrvM3e4A" } + "ifname": { + "type": "unicode", + "value": "if1" + }, + "mac_address": { + "type": "MacAddress", + "value": "qrvM3e4A" + } } }, { "type": "Interface", "value": { - "ifname": { "type": "unicode", "value": "if2" }, - "mac_address": { "type": "MacAddress", "value": "u8zd7v8A" } + "ifname": { + "type": "unicode", + "value": "if2" + }, + "mac_address": { + "type": "MacAddress", + "value": "u8zd7v8A" + } } } ], @@ -2001,25 +3038,37 @@ "type": "unicode", "value": "some-other-hostname.org" }, - "os": { "type": "unicode", "value": "Linux" }, + "os": { + "type": "unicode", + "value": "Linux" + }, "users": [ { "type": "User", "value": { - "username": { "type": "unicode", "value": "user1" } + "username": { + "type": "unicode", + "value": "user1" + } } }, { "type": "User", "value": { - "username": { "type": "unicode", "value": "user2" } + "username": { + "type": "unicode", + "value": "user2" + } } } ] } }, "labels": [], - "memory_size": { "type": "ByteSize", "value": 4294967296 }, + "memory_size": { + "type": "ByteSize", + "value": 4294967296 + }, "os_info": { "type": "Uname", "value": { @@ -2027,10 +3076,22 @@ "type": "unicode", "value": "some-other-hostname.org" }, - "kernel": { "type": "unicode", "value": "4.0.0" }, - "machine": { "type": "unicode", "value": "x86_64" }, - "system": { "type": "unicode", "value": "Linux" }, - "version": { "type": "unicode", "value": "buster/sid" } + "kernel": { + "type": "unicode", + "value": "4.0.0" + }, + "machine": { + "type": "unicode", + "value": "x86_64" + }, + "system": { + "type": "unicode", + "value": "Linux" + }, + "version": { + "type": "unicode", + "value": "buster/sid" + } } }, "urn": { @@ -2041,13 +3102,19 @@ { "type": "User", "value": { - "username": { "type": "unicode", "value": "user1" } + "username": { + "type": "unicode", + "value": "user1" + } } }, { "type": "User", "value": { - "username": { "type": "unicode", "value": "user2" } + "username": { + "type": "unicode", + "value": "user2" + } } } ] @@ -2064,7 +3131,10 @@ "build_time": "1980-01-01", "client_name": "GRR Monitor", "client_version": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "client_id": "C.1000000000000000", "hardware_info": { @@ -2074,7 +3144,10 @@ "interfaces": [ { "addresses": [ - { "address_type": "INET", "packed_bytes": "wKgAAA==" }, + { + "address_type": "INET", + "packed_bytes": "wKgAAA==" + }, { "address_type": "INET6", "packed_bytes": "IAGrzQAAAAAAAAAAAAAAAA==" @@ -2082,13 +3155,26 @@ ], "ifname": "if0" }, - { "ifname": "if1", "mac_address": "qrvM3e4A" }, - { "ifname": "if2", "mac_address": "u8zd7v8A" } + { + "ifname": "if1", + "mac_address": "qrvM3e4A" + }, + { + "ifname": "if2", + "mac_address": "u8zd7v8A" + } ], "knowledge_base": { "fqdn": "Host-0.example.com", "os": "Linux", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] }, "labels": [], "memory_size": 4294967296, @@ -2100,7 +3186,14 @@ "version": "buster/sid" }, "urn": "aff4:/C.1000000000000000", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] }, { "age": 45000000, @@ -2108,7 +3201,10 @@ "build_time": "1980-01-01", "client_name": "GRR Monitor", "client_version": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "client_id": "C.1000000000000000", "hardware_info": { @@ -2118,7 +3214,10 @@ "interfaces": [ { "addresses": [ - { "address_type": "INET", "packed_bytes": "wKgAAA==" }, + { + "address_type": "INET", + "packed_bytes": "wKgAAA==" + }, { "address_type": "INET6", "packed_bytes": "IAGrzQAAAAAAAAAAAAAAAA==" @@ -2126,13 +3225,26 @@ ], "ifname": "if0" }, - { "ifname": "if1", "mac_address": "qrvM3e4A" }, - { "ifname": "if2", "mac_address": "u8zd7v8A" } + { + "ifname": "if1", + "mac_address": "qrvM3e4A" + }, + { + "ifname": "if2", + "mac_address": "u8zd7v8A" + } ], "knowledge_base": { "fqdn": "some-other-hostname.org", "os": "Linux", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] }, "labels": [], "memory_size": 4294967296, @@ -2144,7 +3256,14 @@ "version": "buster/sid" }, "urn": "aff4:/C.1000000000000000", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] } ] }, @@ -2158,16 +3277,34 @@ { "type": "ApiClient", "value": { - "age": { "type": "RDFDatetime", "value": 42000000 }, + "age": { + "type": "RDFDatetime", + "value": 42000000 + }, "agent_info": { "type": "ClientInformation", "value": { - "build_time": { "type": "unicode", "value": "1980-01-01" }, - "client_name": { "type": "unicode", "value": "GRR Monitor" }, - "client_version": { "type": "long", "value": 1234 }, + "build_time": { + "type": "unicode", + "value": "1980-01-01" + }, + "client_name": { + "type": "unicode", + "value": "GRR Monitor" + }, + "client_version": { + "type": "long", + "value": 1234 + }, "labels": [ - { "type": "unicode", "value": "label1" }, - { "type": "unicode", "value": "label2" } + { + "type": "unicode", + "value": "label1" + }, + { + "type": "unicode", + "value": "label2" + } ] } }, @@ -2220,55 +3357,100 @@ } } ], - "ifname": { "type": "unicode", "value": "if0" } + "ifname": { + "type": "unicode", + "value": "if0" + } } }, { "type": "Interface", "value": { - "ifname": { "type": "unicode", "value": "if1" }, - "mac_address": { "type": "MacAddress", "value": "qrvM3e4A" } + "ifname": { + "type": "unicode", + "value": "if1" + }, + "mac_address": { + "type": "MacAddress", + "value": "qrvM3e4A" + } } }, { "type": "Interface", "value": { - "ifname": { "type": "unicode", "value": "if2" }, - "mac_address": { "type": "MacAddress", "value": "u8zd7v8A" } + "ifname": { + "type": "unicode", + "value": "if2" + }, + "mac_address": { + "type": "MacAddress", + "value": "u8zd7v8A" + } } } ], "knowledge_base": { "type": "KnowledgeBase", "value": { - "fqdn": { "type": "unicode", "value": "Host-0.example.com" }, - "os": { "type": "unicode", "value": "Linux" }, + "fqdn": { + "type": "unicode", + "value": "Host-0.example.com" + }, + "os": { + "type": "unicode", + "value": "Linux" + }, "users": [ { "type": "User", "value": { - "username": { "type": "unicode", "value": "user1" } + "username": { + "type": "unicode", + "value": "user1" + } } }, { "type": "User", "value": { - "username": { "type": "unicode", "value": "user2" } + "username": { + "type": "unicode", + "value": "user2" + } } } ] } }, "labels": [], - "memory_size": { "type": "ByteSize", "value": 4294967296 }, + "memory_size": { + "type": "ByteSize", + "value": 4294967296 + }, "os_info": { "type": "Uname", "value": { - "fqdn": { "type": "unicode", "value": "Host-0.example.com" }, - "kernel": { "type": "unicode", "value": "4.0.0" }, - "machine": { "type": "unicode", "value": "x86_64" }, - "system": { "type": "unicode", "value": "Linux" }, - "version": { "type": "unicode", "value": "buster/sid" } + "fqdn": { + "type": "unicode", + "value": "Host-0.example.com" + }, + "kernel": { + "type": "unicode", + "value": "4.0.0" + }, + "machine": { + "type": "unicode", + "value": "x86_64" + }, + "system": { + "type": "unicode", + "value": "Linux" + }, + "version": { + "type": "unicode", + "value": "buster/sid" + } } }, "urn": { @@ -2279,13 +3461,19 @@ { "type": "User", "value": { - "username": { "type": "unicode", "value": "user1" } + "username": { + "type": "unicode", + "value": "user1" + } } }, { "type": "User", "value": { - "username": { "type": "unicode", "value": "user2" } + "username": { + "type": "unicode", + "value": "user2" + } } } ] @@ -2302,7 +3490,10 @@ "build_time": "1980-01-01", "client_name": "GRR Monitor", "client_version": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "client_id": "C.1000000000000000", "hardware_info": { @@ -2312,7 +3503,10 @@ "interfaces": [ { "addresses": [ - { "address_type": "INET", "packed_bytes": "wKgAAA==" }, + { + "address_type": "INET", + "packed_bytes": "wKgAAA==" + }, { "address_type": "INET6", "packed_bytes": "IAGrzQAAAAAAAAAAAAAAAA==" @@ -2320,13 +3514,26 @@ ], "ifname": "if0" }, - { "ifname": "if1", "mac_address": "qrvM3e4A" }, - { "ifname": "if2", "mac_address": "u8zd7v8A" } + { + "ifname": "if1", + "mac_address": "qrvM3e4A" + }, + { + "ifname": "if2", + "mac_address": "u8zd7v8A" + } ], "knowledge_base": { "fqdn": "Host-0.example.com", "os": "Linux", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] }, "labels": [], "memory_size": 4294967296, @@ -2338,7 +3545,14 @@ "version": "buster/sid" }, "urn": "aff4:/C.1000000000000000", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] } ] }, @@ -2352,16 +3566,34 @@ { "type": "ApiClient", "value": { - "age": { "type": "RDFDatetime", "value": 45000000 }, + "age": { + "type": "RDFDatetime", + "value": 45000000 + }, "agent_info": { "type": "ClientInformation", "value": { - "build_time": { "type": "unicode", "value": "1980-01-01" }, - "client_name": { "type": "unicode", "value": "GRR Monitor" }, - "client_version": { "type": "long", "value": 1234 }, + "build_time": { + "type": "unicode", + "value": "1980-01-01" + }, + "client_name": { + "type": "unicode", + "value": "GRR Monitor" + }, + "client_version": { + "type": "long", + "value": 1234 + }, "labels": [ - { "type": "unicode", "value": "label1" }, - { "type": "unicode", "value": "label2" } + { + "type": "unicode", + "value": "label1" + }, + { + "type": "unicode", + "value": "label2" + } ] } }, @@ -2414,21 +3646,36 @@ } } ], - "ifname": { "type": "unicode", "value": "if0" } + "ifname": { + "type": "unicode", + "value": "if0" + } } }, { "type": "Interface", "value": { - "ifname": { "type": "unicode", "value": "if1" }, - "mac_address": { "type": "MacAddress", "value": "qrvM3e4A" } + "ifname": { + "type": "unicode", + "value": "if1" + }, + "mac_address": { + "type": "MacAddress", + "value": "qrvM3e4A" + } } }, { "type": "Interface", "value": { - "ifname": { "type": "unicode", "value": "if2" }, - "mac_address": { "type": "MacAddress", "value": "u8zd7v8A" } + "ifname": { + "type": "unicode", + "value": "if2" + }, + "mac_address": { + "type": "MacAddress", + "value": "u8zd7v8A" + } } } ], @@ -2439,25 +3686,37 @@ "type": "unicode", "value": "some-other-hostname.org" }, - "os": { "type": "unicode", "value": "Linux" }, + "os": { + "type": "unicode", + "value": "Linux" + }, "users": [ { "type": "User", "value": { - "username": { "type": "unicode", "value": "user1" } + "username": { + "type": "unicode", + "value": "user1" + } } }, { "type": "User", "value": { - "username": { "type": "unicode", "value": "user2" } + "username": { + "type": "unicode", + "value": "user2" + } } } ] } }, "labels": [], - "memory_size": { "type": "ByteSize", "value": 4294967296 }, + "memory_size": { + "type": "ByteSize", + "value": 4294967296 + }, "os_info": { "type": "Uname", "value": { @@ -2465,10 +3724,22 @@ "type": "unicode", "value": "some-other-hostname.org" }, - "kernel": { "type": "unicode", "value": "4.0.0" }, - "machine": { "type": "unicode", "value": "x86_64" }, - "system": { "type": "unicode", "value": "Linux" }, - "version": { "type": "unicode", "value": "buster/sid" } + "kernel": { + "type": "unicode", + "value": "4.0.0" + }, + "machine": { + "type": "unicode", + "value": "x86_64" + }, + "system": { + "type": "unicode", + "value": "Linux" + }, + "version": { + "type": "unicode", + "value": "buster/sid" + } } }, "urn": { @@ -2479,13 +3750,19 @@ { "type": "User", "value": { - "username": { "type": "unicode", "value": "user1" } + "username": { + "type": "unicode", + "value": "user1" + } } }, { "type": "User", "value": { - "username": { "type": "unicode", "value": "user2" } + "username": { + "type": "unicode", + "value": "user2" + } } } ] @@ -2502,7 +3779,10 @@ "build_time": "1980-01-01", "client_name": "GRR Monitor", "client_version": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "client_id": "C.1000000000000000", "hardware_info": { @@ -2512,7 +3792,10 @@ "interfaces": [ { "addresses": [ - { "address_type": "INET", "packed_bytes": "wKgAAA==" }, + { + "address_type": "INET", + "packed_bytes": "wKgAAA==" + }, { "address_type": "INET6", "packed_bytes": "IAGrzQAAAAAAAAAAAAAAAA==" @@ -2520,13 +3803,26 @@ ], "ifname": "if0" }, - { "ifname": "if1", "mac_address": "qrvM3e4A" }, - { "ifname": "if2", "mac_address": "u8zd7v8A" } + { + "ifname": "if1", + "mac_address": "qrvM3e4A" + }, + { + "ifname": "if2", + "mac_address": "u8zd7v8A" + } ], "knowledge_base": { "fqdn": "some-other-hostname.org", "os": "Linux", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] }, "labels": [], "memory_size": 4294967296, @@ -2538,7 +3834,14 @@ "version": "buster/sid" }, "urn": "aff4:/C.1000000000000000", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] } ] }, @@ -2552,25 +3855,50 @@ "response": { "type": "ApiCronJobApproval", "value": { - "approvers": [{ "type": "unicode", "value": "api_test_user" }], + "approvers": [ + { + "type": "unicode", + "value": "api_test_user" + } + ], "email_cc_addresses": [], "email_message_id": { "type": "unicode", "value": "" }, - "id": { "type": "unicode", "value": "approval:111111" }, - "is_valid": { "type": "bool", "value": false }, + "id": { + "type": "unicode", + "value": "approval:111111" + }, + "is_valid": { + "type": "bool", + "value": false + }, "is_valid_message": { "type": "unicode", "value": "Need at least 1 additional approver for access." }, - "notified_users": [{ "type": "unicode", "value": "approver" }], - "reason": { "type": "unicode", "value": "foo" }, - "requestor": { "type": "unicode", "value": "api_test_user" }, + "notified_users": [ + { + "type": "unicode", + "value": "approver" + } + ], + "reason": { + "type": "unicode", + "value": "foo" + }, + "requestor": { + "type": "unicode", + "value": "api_test_user" + }, "subject": { "type": "ApiCronJob", "value": { - "allow_overruns": { "type": "bool", "value": false }, + "allow_overruns": { + "type": "bool", + "value": false + }, "args": { "type": "CronJobAction", "value": { @@ -2581,7 +3909,10 @@ "hunt_cron_action": { "type": "HuntCronAction", "value": { - "flow_args": { "type": "InterrogateArgs", "value": {} }, + "flow_args": { + "type": "InterrogateArgs", + "value": {} + }, "flow_name": { "type": "unicode", "value": "Interrogate" @@ -2601,8 +3932,14 @@ "type": "long", "value": 1000 }, - "client_rate": { "type": "float", "value": 20.0 }, - "crash_limit": { "type": "long", "value": 100 } + "client_rate": { + "type": "float", + "value": 20.0 + }, + "crash_limit": { + "type": "long", + "value": 100 + } } } } @@ -2613,23 +3950,39 @@ "type": "ApiCronJobId", "value": "CronJob_123456" }, - "description": { "type": "unicode", "value": "" }, - "enabled": { "type": "bool", "value": true }, - "frequency": { "type": "DurationSeconds", "value": 86400 }, - "is_failing": { "type": "bool", "value": false } + "description": { + "type": "unicode", + "value": "" + }, + "enabled": { + "type": "bool", + "value": true + }, + "frequency": { + "type": "DurationSeconds", + "value": 86400 + }, + "is_failing": { + "type": "bool", + "value": false + } } } } }, "test_class": "ApiGetCronJobApprovalHandlerRegressionTest_http_v1", "type_stripped_response": { - "approvers": ["api_test_user"], + "approvers": [ + "api_test_user" + ], "email_cc_addresses": [], "email_message_id": "", "id": "approval:111111", "is_valid": false, "is_valid_message": "Need at least 1 additional approver for access.", - "notified_users": ["approver"], + "notified_users": [ + "approver" + ], "reason": "foo", "requestor": "api_test_user", "subject": { @@ -2664,23 +4017,49 @@ "type": "ApiCronJobApproval", "value": { "approvers": [ - { "type": "unicode", "value": "api_test_user" }, - { "type": "unicode", "value": "approver" } + { + "type": "unicode", + "value": "api_test_user" + }, + { + "type": "unicode", + "value": "approver" + } ], "email_cc_addresses": [], "email_message_id": { "type": "unicode", "value": "" }, - "id": { "type": "unicode", "value": "approval:222222" }, - "is_valid": { "type": "bool", "value": true }, - "notified_users": [{ "type": "unicode", "value": "approver" }], - "reason": { "type": "unicode", "value": "bar" }, - "requestor": { "type": "unicode", "value": "api_test_user" }, + "id": { + "type": "unicode", + "value": "approval:222222" + }, + "is_valid": { + "type": "bool", + "value": true + }, + "notified_users": [ + { + "type": "unicode", + "value": "approver" + } + ], + "reason": { + "type": "unicode", + "value": "bar" + }, + "requestor": { + "type": "unicode", + "value": "api_test_user" + }, "subject": { "type": "ApiCronJob", "value": { - "allow_overruns": { "type": "bool", "value": false }, + "allow_overruns": { + "type": "bool", + "value": false + }, "args": { "type": "CronJobAction", "value": { @@ -2691,7 +4070,10 @@ "hunt_cron_action": { "type": "HuntCronAction", "value": { - "flow_args": { "type": "InterrogateArgs", "value": {} }, + "flow_args": { + "type": "InterrogateArgs", + "value": {} + }, "flow_name": { "type": "unicode", "value": "Interrogate" @@ -2711,8 +4093,14 @@ "type": "long", "value": 1000 }, - "client_rate": { "type": "float", "value": 20.0 }, - "crash_limit": { "type": "long", "value": 100 } + "client_rate": { + "type": "float", + "value": 20.0 + }, + "crash_limit": { + "type": "long", + "value": 100 + } } } } @@ -2723,22 +4111,39 @@ "type": "ApiCronJobId", "value": "CronJob_567890" }, - "description": { "type": "unicode", "value": "" }, - "enabled": { "type": "bool", "value": true }, - "frequency": { "type": "DurationSeconds", "value": 86400 }, - "is_failing": { "type": "bool", "value": false } + "description": { + "type": "unicode", + "value": "" + }, + "enabled": { + "type": "bool", + "value": true + }, + "frequency": { + "type": "DurationSeconds", + "value": 86400 + }, + "is_failing": { + "type": "bool", + "value": false + } } } } }, "test_class": "ApiGetCronJobApprovalHandlerRegressionTest_http_v1", "type_stripped_response": { - "approvers": ["api_test_user", "approver"], + "approvers": [ + "api_test_user", + "approver" + ], "email_cc_addresses": [], "email_message_id": "", "id": "approval:222222", "is_valid": true, - "notified_users": ["approver"], + "notified_users": [ + "approver" + ], "reason": "bar", "requestor": "api_test_user", "subject": { @@ -2778,10 +4183,22 @@ "type": "ApiCronJobId", "value": "GRRVersionBreakDown" }, - "finished_at": { "type": "RDFDatetime", "value": 44000000 }, - "run_id": { "type": "ApiCronJobRunId", "value": "F:ABCDEF11" }, - "started_at": { "type": "RDFDatetime", "value": 44000000 }, - "status": { "type": "EnumNamedValue", "value": "FINISHED" } + "finished_at": { + "type": "RDFDatetime", + "value": 44000000 + }, + "run_id": { + "type": "ApiCronJobRunId", + "value": "F:ABCDEF11" + }, + "started_at": { + "type": "RDFDatetime", + "value": 44000000 + }, + "status": { + "type": "EnumNamedValue", + "value": "FINISHED" + } } }, "test_class": "ApiGetCronJobRunHandlerRegressionTest_http_v1", @@ -2810,21 +4227,30 @@ { "api_method": "GetFileText", "method": "GET", - "response": { "content": "Goodbye World", "total_size": 13 }, + "response": { + "content": "Goodbye World", + "total_size": 13 + }, "test_class": "ApiGetFileTextHandlerRegressionTest_http_v1", "url": "/api/clients/C.1000000000000000/vfs-text/fs/os/c/Downloads/a.txt" }, { "api_method": "GetFileText", "method": "GET", - "response": { "content": "dby", "total_size": 13 }, + "response": { + "content": "dby", + "total_size": 13 + }, "test_class": "ApiGetFileTextHandlerRegressionTest_http_v1", "url": "/api/clients/C.1000000000000000/vfs-text/fs/os/c/Downloads/a.txt?length=3&offset=3" }, { "api_method": "GetFileText", "method": "GET", - "response": { "content": "Hello World", "total_size": 11 }, + "response": { + "content": "Hello World", + "total_size": 11 + }, "test_class": "ApiGetFileTextHandlerRegressionTest_http_v1", "url": "/api/clients/C.1000000000000000/vfs-text/fs/os/c/Downloads/a.txt?timestamp=86400000042" } @@ -2835,12 +4261,23 @@ "method": "GET", "response": { "times": [ - { "type": "RDFDatetime", "value": 172800000042 }, - { "type": "RDFDatetime", "value": 86400000042 } + { + "type": "RDFDatetime", + "value": 172800000042 + }, + { + "type": "RDFDatetime", + "value": 86400000042 + } ] }, "test_class": "ApiGetFileVersionTimesHandlerRegressionTest_http_v1", - "type_stripped_response": { "times": [172800000042, 86400000042] }, + "type_stripped_response": { + "times": [ + 172800000042, + 86400000042 + ] + }, "url": "/api/clients/C.1000000000000000/vfs-version-times/fs/os/c/Downloads/a.txt" } ], @@ -2851,50 +4288,122 @@ "response": { "type": "ApiFlow", "value": { - "args": { "type": "InterrogateArgs", "value": {} }, - "client_id": { "type": "ApiClientId", "value": "C.1000000000000000" }, + "args": { + "type": "InterrogateArgs", + "value": {} + }, + "client_id": { + "type": "ApiClientId", + "value": "C.1000000000000000" + }, "context": { "type": "FlowContext", "value": { "client_resources": { "type": "ClientResources", - "value": { "cpu_usage": { "type": "CpuSeconds", "value": {} } } + "value": { + "cpu_usage": { + "type": "CpuSeconds", + "value": {} + } + } + }, + "create_time": { + "type": "RDFDatetime", + "value": 42000000 + }, + "creator": { + "type": "unicode", + "value": "api_test_user" + }, + "current_state": { + "type": "unicode", + "value": "Start" + }, + "next_outbound_id": { + "type": "long", + "value": 9 + }, + "outstanding_requests": { + "type": "long", + "value": 8 }, - "create_time": { "type": "RDFDatetime", "value": 42000000 }, - "creator": { "type": "unicode", "value": "api_test_user" }, - "current_state": { "type": "unicode", "value": "Start" }, - "next_outbound_id": { "type": "long", "value": 9 }, - "outstanding_requests": { "type": "long", "value": 8 }, "session_id": { "type": "SessionID", "value": "aff4:/C.1000000000000000/flows/F:ABCDEF12" }, - "state": { "type": "EnumNamedValue", "value": "RUNNING" } + "state": { + "type": "EnumNamedValue", + "value": "RUNNING" + } } }, - "creator": { "type": "unicode", "value": "api_test_user" }, - "flow_id": { "type": "ApiFlowId", "value": "F:ABCDEF12" }, - "is_robot": { "type": "bool", "value": false }, - "last_active_at": { "type": "RDFDatetime", "value": 42000000 }, - "name": { "type": "unicode", "value": "Interrogate" }, - "progress": { "type": "DefaultFlowProgress", "value": {} }, - "result_metadata": { - "type": "FlowResultMetadata", - "value": { "is_metadata_set": { "type": "bool", "value": true } } + "creator": { + "type": "unicode", + "value": "api_test_user" }, - "runner_args": { - "type": "FlowRunnerArgs", - "value": { - "client_id": { + "flow_id": { + "type": "ApiFlowId", + "value": "F:ABCDEF12" + }, + "is_robot": { + "type": "bool", + "value": false + }, + "last_active_at": { + "type": "RDFDatetime", + "value": 42000000 + }, + "name": { + "type": "unicode", + "value": "Interrogate" + }, + "progress": { + "type": "DefaultFlowProgress", + "value": {} + }, + "result_metadata": { + "type": "FlowResultMetadata", + "value": { + "is_metadata_set": { + "type": "bool", + "value": true + } + } + }, + "runner_args": { + "type": "FlowRunnerArgs", + "value": { + "client_id": { "type": "ClientURN", "value": "aff4:/C.1000000000000000" }, - "flow_name": { "type": "unicode", "value": "Interrogate" }, - "notify_to_user": { "type": "bool", "value": true } + "cpu_limit": { + "type": "long", + "value": 60 + }, + "flow_name": { + "type": "unicode", + "value": "Interrogate" + }, + "network_bytes_limit": { + "type": "long", + "value": 8192 + }, + "notify_to_user": { + "type": "bool", + "value": true + } } }, - "started_at": { "type": "RDFDatetime", "value": 42000000 }, - "state": { "type": "EnumNamedValue", "value": "RUNNING" }, + "started_at": { + "type": "RDFDatetime", + "value": 42000000 + }, + "state": { + "type": "EnumNamedValue", + "value": "RUNNING" + }, "state_data": { "type": "ApiDataObject", "value": { @@ -2902,7 +4411,10 @@ { "type": "ApiDataObjectKeyValuePair", "value": { - "key": { "type": "unicode", "value": "_result_metadata" }, + "key": { + "type": "unicode", + "value": "_result_metadata" + }, "type": { "type": "unicode", "value": "FlowResultMetadata" @@ -2910,7 +4422,10 @@ "value": { "type": "FlowResultMetadata", "value": { - "is_metadata_set": { "type": "bool", "value": true } + "is_metadata_set": { + "type": "bool", + "value": true + } } } } @@ -2918,8 +4433,14 @@ { "type": "ApiDataObjectKeyValuePair", "value": { - "key": { "type": "unicode", "value": "client" }, - "type": { "type": "unicode", "value": "ClientSnapshot" }, + "key": { + "type": "unicode", + "value": "client" + }, + "type": { + "type": "unicode", + "value": "ClientSnapshot" + }, "value": { "type": "ClientSnapshot", "value": { @@ -2943,15 +4464,27 @@ { "type": "ApiDataObjectKeyValuePair", "value": { - "invalid": { "type": "bool", "value": true }, - "key": { "type": "unicode", "value": "fqdn" } + "invalid": { + "type": "bool", + "value": true + }, + "key": { + "type": "unicode", + "value": "fqdn" + } } }, { "type": "ApiDataObjectKeyValuePair", "value": { - "invalid": { "type": "bool", "value": true }, - "key": { "type": "unicode", "value": "os" } + "invalid": { + "type": "bool", + "value": true + }, + "key": { + "type": "unicode", + "value": "os" + } } } ] @@ -2968,7 +4501,9 @@ "args": {}, "client_id": "C.1000000000000000", "context": { - "client_resources": { "cpu_usage": {} }, + "client_resources": { + "cpu_usage": {} + }, "create_time": 42000000, "creator": "api_test_user", "current_state": "Start", @@ -2983,23 +4518,37 @@ "last_active_at": 42000000, "name": "Interrogate", "progress": {}, - "result_metadata": { "is_metadata_set": true }, + "result_metadata": { + "is_metadata_set": true + }, "runner_args": { "client_id": "aff4:/C.1000000000000000", + "cpu_limit": 60, "flow_name": "Interrogate", + "network_bytes_limit": 8192, "notify_to_user": true }, "started_at": 42000000, "state": "RUNNING", "state_data": { "items": [ - { "is_metadata_set": true }, + { + "is_metadata_set": true + }, { "client_id": "C.1000000000000000", - "metadata": { "source_flow_id": "F:ABCDEF12" } + "metadata": { + "source_flow_id": "F:ABCDEF12" + } }, - { "invalid": true, "key": "fqdn" }, - { "invalid": true, "key": "os" } + { + "invalid": true, + "key": "fqdn" + }, + { + "invalid": true, + "key": "os" + } ] }, "urn": "aff4:/C.1000000000000000/flows/F:ABCDEF12" @@ -3012,44 +4561,96 @@ "response": { "type": "ApiFlow", "value": { - "args": { "type": "InterrogateArgs", "value": {} }, - "client_id": { "type": "ApiClientId", "value": "C.1000000000000000" }, + "args": { + "type": "InterrogateArgs", + "value": {} + }, + "client_id": { + "type": "ApiClientId", + "value": "C.1000000000000000" + }, "context": { "type": "FlowContext", "value": { "client_resources": { "type": "ClientResources", - "value": { "cpu_usage": { "type": "CpuSeconds", "value": {} } } + "value": { + "cpu_usage": { + "type": "CpuSeconds", + "value": {} + } + } + }, + "create_time": { + "type": "RDFDatetime", + "value": 42000000 + }, + "creator": { + "type": "unicode", + "value": "api_test_user" + }, + "current_state": { + "type": "unicode", + "value": "Start" + }, + "next_outbound_id": { + "type": "long", + "value": 9 + }, + "outstanding_requests": { + "type": "long", + "value": 8 }, - "create_time": { "type": "RDFDatetime", "value": 42000000 }, - "creator": { "type": "unicode", "value": "api_test_user" }, - "current_state": { "type": "unicode", "value": "Start" }, - "next_outbound_id": { "type": "long", "value": 9 }, - "outstanding_requests": { "type": "long", "value": 8 }, "session_id": { "type": "SessionID", "value": "aff4:/C.1000000000000000/flows/F:ABCDEF13" }, - "state": { "type": "EnumNamedValue", "value": "ERROR" }, + "state": { + "type": "EnumNamedValue", + "value": "ERROR" + }, "status": { "type": "unicode", "value": "Pending termination: Some reason" } } }, - "creator": { "type": "unicode", "value": "api_test_user" }, + "creator": { + "type": "unicode", + "value": "api_test_user" + }, "error_description": { "type": "unicode", "value": "Pending termination: Some reason" }, - "flow_id": { "type": "ApiFlowId", "value": "F:ABCDEF13" }, - "is_robot": { "type": "bool", "value": false }, - "last_active_at": { "type": "RDFDatetime", "value": 42000000 }, - "name": { "type": "unicode", "value": "Interrogate" }, - "progress": { "type": "DefaultFlowProgress", "value": {} }, + "flow_id": { + "type": "ApiFlowId", + "value": "F:ABCDEF13" + }, + "is_robot": { + "type": "bool", + "value": false + }, + "last_active_at": { + "type": "RDFDatetime", + "value": 42000000 + }, + "name": { + "type": "unicode", + "value": "Interrogate" + }, + "progress": { + "type": "DefaultFlowProgress", + "value": {} + }, "result_metadata": { "type": "FlowResultMetadata", - "value": { "is_metadata_set": { "type": "bool", "value": true } } + "value": { + "is_metadata_set": { + "type": "bool", + "value": true + } + } }, "runner_args": { "type": "FlowRunnerArgs", @@ -3058,12 +4659,32 @@ "type": "ClientURN", "value": "aff4:/C.1000000000000000" }, - "flow_name": { "type": "unicode", "value": "Interrogate" }, - "notify_to_user": { "type": "bool", "value": true } + "cpu_limit": { + "type": "long", + "value": 60 + }, + "flow_name": { + "type": "unicode", + "value": "Interrogate" + }, + "network_bytes_limit": { + "type": "long", + "value": 8192 + }, + "notify_to_user": { + "type": "bool", + "value": true + } } }, - "started_at": { "type": "RDFDatetime", "value": 42000000 }, - "state": { "type": "EnumNamedValue", "value": "ERROR" }, + "started_at": { + "type": "RDFDatetime", + "value": 42000000 + }, + "state": { + "type": "EnumNamedValue", + "value": "ERROR" + }, "state_data": { "type": "ApiDataObject", "value": { @@ -3071,7 +4692,10 @@ { "type": "ApiDataObjectKeyValuePair", "value": { - "key": { "type": "unicode", "value": "_result_metadata" }, + "key": { + "type": "unicode", + "value": "_result_metadata" + }, "type": { "type": "unicode", "value": "FlowResultMetadata" @@ -3079,7 +4703,10 @@ "value": { "type": "FlowResultMetadata", "value": { - "is_metadata_set": { "type": "bool", "value": true } + "is_metadata_set": { + "type": "bool", + "value": true + } } } } @@ -3087,8 +4714,14 @@ { "type": "ApiDataObjectKeyValuePair", "value": { - "key": { "type": "unicode", "value": "client" }, - "type": { "type": "unicode", "value": "ClientSnapshot" }, + "key": { + "type": "unicode", + "value": "client" + }, + "type": { + "type": "unicode", + "value": "ClientSnapshot" + }, "value": { "type": "ClientSnapshot", "value": { @@ -3112,15 +4745,27 @@ { "type": "ApiDataObjectKeyValuePair", "value": { - "invalid": { "type": "bool", "value": true }, - "key": { "type": "unicode", "value": "fqdn" } + "invalid": { + "type": "bool", + "value": true + }, + "key": { + "type": "unicode", + "value": "fqdn" + } } }, { "type": "ApiDataObjectKeyValuePair", "value": { - "invalid": { "type": "bool", "value": true }, - "key": { "type": "unicode", "value": "os" } + "invalid": { + "type": "bool", + "value": true + }, + "key": { + "type": "unicode", + "value": "os" + } } } ] @@ -3137,7 +4782,9 @@ "args": {}, "client_id": "C.1000000000000000", "context": { - "client_resources": { "cpu_usage": {} }, + "client_resources": { + "cpu_usage": {} + }, "create_time": 42000000, "creator": "api_test_user", "current_state": "Start", @@ -3154,23 +4801,37 @@ "last_active_at": 42000000, "name": "Interrogate", "progress": {}, - "result_metadata": { "is_metadata_set": true }, + "result_metadata": { + "is_metadata_set": true + }, "runner_args": { "client_id": "aff4:/C.1000000000000000", + "cpu_limit": 60, "flow_name": "Interrogate", + "network_bytes_limit": 8192, "notify_to_user": true }, "started_at": 42000000, "state": "ERROR", "state_data": { "items": [ - { "is_metadata_set": true }, + { + "is_metadata_set": true + }, { "client_id": "C.1000000000000000", - "metadata": { "source_flow_id": "F:ABCDEF13" } + "metadata": { + "source_flow_id": "F:ABCDEF13" + } + }, + { + "invalid": true, + "key": "fqdn" }, - { "invalid": true, "key": "fqdn" }, - { "invalid": true, "key": "os" } + { + "invalid": true, + "key": "os" + } ] }, "urn": "aff4:/C.1000000000000000/flows/F:ABCDEF13" @@ -3240,7 +4901,12 @@ "response": { "type": "ApiHuntApproval", "value": { - "approvers": [{ "type": "unicode", "value": "api_test_user" }], + "approvers": [ + { + "type": "unicode", + "value": "api_test_user" + } + ], "email_cc_addresses": [], "email_message_id": { "type": "unicode", @@ -3250,21 +4916,47 @@ "type": "RDFDatetime", "value": 2419244000000 }, - "id": { "type": "unicode", "value": "approval:111111" }, - "is_valid": { "type": "bool", "value": false }, + "id": { + "type": "unicode", + "value": "approval:111111" + }, + "is_valid": { + "type": "bool", + "value": false + }, "is_valid_message": { "type": "unicode", "value": "Need at least 1 additional approver for access." }, - "notified_users": [{ "type": "unicode", "value": "approver" }], - "reason": { "type": "unicode", "value": "foo" }, - "requestor": { "type": "unicode", "value": "api_test_user" }, + "notified_users": [ + { + "type": "unicode", + "value": "approver" + } + ], + "reason": { + "type": "unicode", + "value": "foo" + }, + "requestor": { + "type": "unicode", + "value": "api_test_user" + }, "subject": { "type": "ApiHunt", "value": { - "all_clients_count": { "type": "long", "value": 0 }, - "client_limit": { "type": "long", "value": 100 }, - "client_rate": { "type": "float", "value": 0.0 }, + "all_clients_count": { + "type": "long", + "value": 0 + }, + "client_limit": { + "type": "long", + "value": 100 + }, + "client_rate": { + "type": "float", + "value": 0.0 + }, "client_rule_set": { "type": "ForemanClientRuleSet", "value": { @@ -3294,29 +4986,68 @@ ] } }, - "clients_with_results_count": { "type": "long", "value": 0 }, - "completed_clients_count": { "type": "long", "value": 0 }, - "crash_limit": { "type": "long", "value": 100 }, - "crashed_clients_count": { "type": "long", "value": 0 }, - "created": { "type": "RDFDatetime", "value": 42000000 }, - "creator": { "type": "unicode", "value": "api_test_user" }, - "description": { "type": "unicode", "value": "hunt1" }, - "duration": { "type": "DurationSeconds", "value": 1209600 }, - "failed_clients_count": { "type": "long", "value": 0 }, + "clients_with_results_count": { + "type": "long", + "value": 0 + }, + "completed_clients_count": { + "type": "long", + "value": 0 + }, + "crash_limit": { + "type": "long", + "value": 100 + }, + "crashed_clients_count": { + "type": "long", + "value": 0 + }, + "created": { + "type": "RDFDatetime", + "value": 42000000 + }, + "creator": { + "type": "unicode", + "value": "api_test_user" + }, + "description": { + "type": "unicode", + "value": "hunt1" + }, + "duration": { + "type": "DurationSeconds", + "value": 1209600 + }, + "failed_clients_count": { + "type": "long", + "value": 0 + }, "flow_args": { "type": "GetFileArgs", "value": { "pathspec": { "type": "PathSpec", "value": { - "path": { "type": "unicode", "value": "/tmp/evil.txt" }, - "pathtype": { "type": "EnumNamedValue", "value": "OS" } + "path": { + "type": "unicode", + "value": "/tmp/evil.txt" + }, + "pathtype": { + "type": "EnumNamedValue", + "value": "OS" + } } } } }, - "flow_name": { "type": "unicode", "value": "GetFile" }, - "hunt_id": { "type": "ApiHuntId", "value": "H:123456" }, + "flow_name": { + "type": "unicode", + "value": "GetFile" + }, + "hunt_id": { + "type": "ApiHuntId", + "value": "H:123456" + }, "hunt_runner_args": { "type": "HuntRunnerArgs", "value": { @@ -3332,7 +5063,10 @@ "type": "long", "value": 1000 }, - "client_rate": { "type": "float", "value": 0.0 }, + "client_rate": { + "type": "float", + "value": 0.0 + }, "client_rule_set": { "type": "ForemanClientRuleSet", "value": { @@ -3362,43 +5096,90 @@ ] } }, - "crash_limit": { "type": "long", "value": 100 }, - "description": { "type": "unicode", "value": "hunt1" }, + "crash_limit": { + "type": "long", + "value": 100 + }, + "description": { + "type": "unicode", + "value": "hunt1" + }, "expiry_time": { "type": "DurationSeconds", "value": 1209600 }, - "hunt_name": { "type": "unicode", "value": "GenericHunt" }, + "hunt_name": { + "type": "unicode", + "value": "GenericHunt" + }, "original_object": { "type": "FlowLikeObjectReference", "value": {} } } }, - "hunt_type": { "type": "EnumNamedValue", "value": "STANDARD" }, - "is_robot": { "type": "bool", "value": false }, - "name": { "type": "unicode", "value": "GenericHunt" }, - "remaining_clients_count": { "type": "long", "value": 0 }, - "results_count": { "type": "long", "value": 0 }, - "state": { "type": "EnumNamedValue", "value": "PAUSED" }, - "state_comment": { "type": "unicode", "value": "" }, - "total_cpu_usage": { "type": "long", "value": 0 }, - "total_net_usage": { "type": "long", "value": 0 }, - "urn": { "type": "SessionID", "value": "aff4:/hunts/H:123456" } + "hunt_type": { + "type": "EnumNamedValue", + "value": "STANDARD" + }, + "is_robot": { + "type": "bool", + "value": false + }, + "name": { + "type": "unicode", + "value": "GenericHunt" + }, + "remaining_clients_count": { + "type": "long", + "value": 0 + }, + "results_count": { + "type": "long", + "value": 0 + }, + "state": { + "type": "EnumNamedValue", + "value": "PAUSED" + }, + "state_comment": { + "type": "unicode", + "value": "" + }, + "state_reason": { + "type": "EnumNamedValue", + "value": "UNKNOWN" + }, + "total_cpu_usage": { + "type": "long", + "value": 0 + }, + "total_net_usage": { + "type": "long", + "value": 0 + }, + "urn": { + "type": "SessionID", + "value": "aff4:/hunts/H:123456" + } } } } }, "test_class": "ApiGetHuntApprovalHandlerRegressionTest_http_v1", "type_stripped_response": { - "approvers": ["api_test_user"], + "approvers": [ + "api_test_user" + ], "email_cc_addresses": [], "email_message_id": "", "expiration_time_us": 2419244000000, "id": "approval:111111", "is_valid": false, "is_valid_message": "Need at least 1 additional approver for access.", - "notified_users": ["approver"], + "notified_users": [ + "approver" + ], "reason": "foo", "requestor": "api_test_user", "subject": { @@ -3408,7 +5189,10 @@ "client_rule_set": { "rules": [ { - "regex": { "attribute_regex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attribute_regex": "GRR", + "field": "CLIENT_NAME" + }, "rule_type": "REGEX" } ] @@ -3423,7 +5207,10 @@ "duration": 1209600, "failed_clients_count": 0, "flow_args": { - "pathspec": { "path": "/tmp/evil.txt", "pathtype": "OS" } + "pathspec": { + "path": "/tmp/evil.txt", + "pathtype": "OS" + } }, "flow_name": "GetFile", "hunt_id": "H:123456", @@ -3435,7 +5222,10 @@ "client_rule_set": { "rules": [ { - "regex": { "attribute_regex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attribute_regex": "GRR", + "field": "CLIENT_NAME" + }, "rule_type": "REGEX" } ] @@ -3453,6 +5243,7 @@ "results_count": 0, "state": "PAUSED", "state_comment": "", + "state_reason": "UNKNOWN", "total_cpu_usage": 0, "total_net_usage": 0, "urn": "aff4:/hunts/H:123456" @@ -3467,8 +5258,14 @@ "type": "ApiHuntApproval", "value": { "approvers": [ - { "type": "unicode", "value": "api_test_user" }, - { "type": "unicode", "value": "approver" } + { + "type": "unicode", + "value": "api_test_user" + }, + { + "type": "unicode", + "value": "approver" + } ], "email_cc_addresses": [], "email_message_id": { @@ -3479,17 +5276,43 @@ "type": "RDFDatetime", "value": 2419245000000 }, - "id": { "type": "unicode", "value": "approval:222222" }, - "is_valid": { "type": "bool", "value": true }, - "notified_users": [{ "type": "unicode", "value": "approver" }], - "reason": { "type": "unicode", "value": "bar" }, - "requestor": { "type": "unicode", "value": "api_test_user" }, + "id": { + "type": "unicode", + "value": "approval:222222" + }, + "is_valid": { + "type": "bool", + "value": true + }, + "notified_users": [ + { + "type": "unicode", + "value": "approver" + } + ], + "reason": { + "type": "unicode", + "value": "bar" + }, + "requestor": { + "type": "unicode", + "value": "api_test_user" + }, "subject": { "type": "ApiHunt", "value": { - "all_clients_count": { "type": "long", "value": 0 }, - "client_limit": { "type": "long", "value": 100 }, - "client_rate": { "type": "float", "value": 0.0 }, + "all_clients_count": { + "type": "long", + "value": 0 + }, + "client_limit": { + "type": "long", + "value": 100 + }, + "client_rate": { + "type": "float", + "value": 0.0 + }, "client_rule_set": { "type": "ForemanClientRuleSet", "value": { @@ -3519,29 +5342,68 @@ ] } }, - "clients_with_results_count": { "type": "long", "value": 0 }, - "completed_clients_count": { "type": "long", "value": 0 }, - "crash_limit": { "type": "long", "value": 100 }, - "crashed_clients_count": { "type": "long", "value": 0 }, - "created": { "type": "RDFDatetime", "value": 42000000 }, - "creator": { "type": "unicode", "value": "api_test_user" }, - "description": { "type": "unicode", "value": "hunt2" }, - "duration": { "type": "DurationSeconds", "value": 1209600 }, - "failed_clients_count": { "type": "long", "value": 0 }, + "clients_with_results_count": { + "type": "long", + "value": 0 + }, + "completed_clients_count": { + "type": "long", + "value": 0 + }, + "crash_limit": { + "type": "long", + "value": 100 + }, + "crashed_clients_count": { + "type": "long", + "value": 0 + }, + "created": { + "type": "RDFDatetime", + "value": 42000000 + }, + "creator": { + "type": "unicode", + "value": "api_test_user" + }, + "description": { + "type": "unicode", + "value": "hunt2" + }, + "duration": { + "type": "DurationSeconds", + "value": 1209600 + }, + "failed_clients_count": { + "type": "long", + "value": 0 + }, "flow_args": { "type": "GetFileArgs", "value": { "pathspec": { "type": "PathSpec", "value": { - "path": { "type": "unicode", "value": "/tmp/evil.txt" }, - "pathtype": { "type": "EnumNamedValue", "value": "OS" } - } - } + "path": { + "type": "unicode", + "value": "/tmp/evil.txt" + }, + "pathtype": { + "type": "EnumNamedValue", + "value": "OS" + } + } + } } }, - "flow_name": { "type": "unicode", "value": "GetFile" }, - "hunt_id": { "type": "ApiHuntId", "value": "H:567890" }, + "flow_name": { + "type": "unicode", + "value": "GetFile" + }, + "hunt_id": { + "type": "ApiHuntId", + "value": "H:567890" + }, "hunt_runner_args": { "type": "HuntRunnerArgs", "value": { @@ -3557,7 +5419,10 @@ "type": "long", "value": 1000 }, - "client_rate": { "type": "float", "value": 0.0 }, + "client_rate": { + "type": "float", + "value": 0.0 + }, "client_rule_set": { "type": "ForemanClientRuleSet", "value": { @@ -3587,42 +5452,90 @@ ] } }, - "crash_limit": { "type": "long", "value": 100 }, - "description": { "type": "unicode", "value": "hunt2" }, + "crash_limit": { + "type": "long", + "value": 100 + }, + "description": { + "type": "unicode", + "value": "hunt2" + }, "expiry_time": { "type": "DurationSeconds", "value": 1209600 }, - "hunt_name": { "type": "unicode", "value": "GenericHunt" }, + "hunt_name": { + "type": "unicode", + "value": "GenericHunt" + }, "original_object": { "type": "FlowLikeObjectReference", "value": {} } } }, - "hunt_type": { "type": "EnumNamedValue", "value": "STANDARD" }, - "is_robot": { "type": "bool", "value": false }, - "name": { "type": "unicode", "value": "GenericHunt" }, - "remaining_clients_count": { "type": "long", "value": 0 }, - "results_count": { "type": "long", "value": 0 }, - "state": { "type": "EnumNamedValue", "value": "PAUSED" }, - "state_comment": { "type": "unicode", "value": "" }, - "total_cpu_usage": { "type": "long", "value": 0 }, - "total_net_usage": { "type": "long", "value": 0 }, - "urn": { "type": "SessionID", "value": "aff4:/hunts/H:567890" } + "hunt_type": { + "type": "EnumNamedValue", + "value": "STANDARD" + }, + "is_robot": { + "type": "bool", + "value": false + }, + "name": { + "type": "unicode", + "value": "GenericHunt" + }, + "remaining_clients_count": { + "type": "long", + "value": 0 + }, + "results_count": { + "type": "long", + "value": 0 + }, + "state": { + "type": "EnumNamedValue", + "value": "PAUSED" + }, + "state_comment": { + "type": "unicode", + "value": "" + }, + "state_reason": { + "type": "EnumNamedValue", + "value": "UNKNOWN" + }, + "total_cpu_usage": { + "type": "long", + "value": 0 + }, + "total_net_usage": { + "type": "long", + "value": 0 + }, + "urn": { + "type": "SessionID", + "value": "aff4:/hunts/H:567890" + } } } } }, "test_class": "ApiGetHuntApprovalHandlerRegressionTest_http_v1", "type_stripped_response": { - "approvers": ["api_test_user", "approver"], + "approvers": [ + "api_test_user", + "approver" + ], "email_cc_addresses": [], "email_message_id": "", "expiration_time_us": 2419245000000, "id": "approval:222222", "is_valid": true, - "notified_users": ["approver"], + "notified_users": [ + "approver" + ], "reason": "bar", "requestor": "api_test_user", "subject": { @@ -3632,7 +5545,10 @@ "client_rule_set": { "rules": [ { - "regex": { "attribute_regex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attribute_regex": "GRR", + "field": "CLIENT_NAME" + }, "rule_type": "REGEX" } ] @@ -3647,7 +5563,10 @@ "duration": 1209600, "failed_clients_count": 0, "flow_args": { - "pathspec": { "path": "/tmp/evil.txt", "pathtype": "OS" } + "pathspec": { + "path": "/tmp/evil.txt", + "pathtype": "OS" + } }, "flow_name": "GetFile", "hunt_id": "H:567890", @@ -3659,7 +5578,10 @@ "client_rule_set": { "rules": [ { - "regex": { "attribute_regex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attribute_regex": "GRR", + "field": "CLIENT_NAME" + }, "rule_type": "REGEX" } ] @@ -3677,6 +5599,7 @@ "results_count": 0, "state": "PAUSED", "state_comment": "", + "state_reason": "UNKNOWN", "total_cpu_usage": 0, "total_net_usage": 0, "urn": "aff4:/hunts/H:567890" @@ -3690,13 +5613,27 @@ "response": { "type": "ApiHuntApproval", "value": { - "approvers": [{ "type": "unicode", "value": "api_test_user" }], + "approvers": [ + { + "type": "unicode", + "value": "api_test_user" + } + ], "copied_from_hunt": { "type": "ApiHunt", "value": { - "all_clients_count": { "type": "long", "value": 0 }, - "client_limit": { "type": "long", "value": 100 }, - "client_rate": { "type": "float", "value": 0.0 }, + "all_clients_count": { + "type": "long", + "value": 0 + }, + "client_limit": { + "type": "long", + "value": 100 + }, + "client_rate": { + "type": "float", + "value": 0.0 + }, "client_rule_set": { "type": "ForemanClientRuleSet", "value": { @@ -3726,29 +5663,68 @@ ] } }, - "clients_with_results_count": { "type": "long", "value": 0 }, - "completed_clients_count": { "type": "long", "value": 0 }, - "crash_limit": { "type": "long", "value": 100 }, - "crashed_clients_count": { "type": "long", "value": 0 }, - "created": { "type": "RDFDatetime", "value": 42000000 }, - "creator": { "type": "unicode", "value": "api_test_user" }, - "description": { "type": "unicode", "value": "original hunt" }, - "duration": { "type": "DurationSeconds", "value": 1209600 }, - "failed_clients_count": { "type": "long", "value": 0 }, + "clients_with_results_count": { + "type": "long", + "value": 0 + }, + "completed_clients_count": { + "type": "long", + "value": 0 + }, + "crash_limit": { + "type": "long", + "value": 100 + }, + "crashed_clients_count": { + "type": "long", + "value": 0 + }, + "created": { + "type": "RDFDatetime", + "value": 42000000 + }, + "creator": { + "type": "unicode", + "value": "api_test_user" + }, + "description": { + "type": "unicode", + "value": "original hunt" + }, + "duration": { + "type": "DurationSeconds", + "value": 1209600 + }, + "failed_clients_count": { + "type": "long", + "value": 0 + }, "flow_args": { "type": "GetFileArgs", "value": { "pathspec": { "type": "PathSpec", "value": { - "path": { "type": "unicode", "value": "/tmp/evil.txt" }, - "pathtype": { "type": "EnumNamedValue", "value": "OS" } + "path": { + "type": "unicode", + "value": "/tmp/evil.txt" + }, + "pathtype": { + "type": "EnumNamedValue", + "value": "OS" + } } } } }, - "flow_name": { "type": "unicode", "value": "GetFile" }, - "hunt_id": { "type": "ApiHuntId", "value": "H:556677" }, + "flow_name": { + "type": "unicode", + "value": "GetFile" + }, + "hunt_id": { + "type": "ApiHuntId", + "value": "H:556677" + }, "hunt_runner_args": { "type": "HuntRunnerArgs", "value": { @@ -3764,7 +5740,10 @@ "type": "long", "value": 1000 }, - "client_rate": { "type": "float", "value": 0.0 }, + "client_rate": { + "type": "float", + "value": 0.0 + }, "client_rule_set": { "type": "ForemanClientRuleSet", "value": { @@ -3794,7 +5773,10 @@ ] } }, - "crash_limit": { "type": "long", "value": 100 }, + "crash_limit": { + "type": "long", + "value": 100 + }, "description": { "type": "unicode", "value": "original hunt" @@ -3803,23 +5785,60 @@ "type": "DurationSeconds", "value": 1209600 }, - "hunt_name": { "type": "unicode", "value": "GenericHunt" }, + "hunt_name": { + "type": "unicode", + "value": "GenericHunt" + }, "original_object": { "type": "FlowLikeObjectReference", "value": {} } } }, - "hunt_type": { "type": "EnumNamedValue", "value": "STANDARD" }, - "is_robot": { "type": "bool", "value": false }, - "name": { "type": "unicode", "value": "GenericHunt" }, - "remaining_clients_count": { "type": "long", "value": 0 }, - "results_count": { "type": "long", "value": 0 }, - "state": { "type": "EnumNamedValue", "value": "PAUSED" }, - "state_comment": { "type": "unicode", "value": "" }, - "total_cpu_usage": { "type": "long", "value": 0 }, - "total_net_usage": { "type": "long", "value": 0 }, - "urn": { "type": "SessionID", "value": "aff4:/hunts/H:556677" } + "hunt_type": { + "type": "EnumNamedValue", + "value": "STANDARD" + }, + "is_robot": { + "type": "bool", + "value": false + }, + "name": { + "type": "unicode", + "value": "GenericHunt" + }, + "remaining_clients_count": { + "type": "long", + "value": 0 + }, + "results_count": { + "type": "long", + "value": 0 + }, + "state": { + "type": "EnumNamedValue", + "value": "PAUSED" + }, + "state_comment": { + "type": "unicode", + "value": "" + }, + "state_reason": { + "type": "EnumNamedValue", + "value": "UNKNOWN" + }, + "total_cpu_usage": { + "type": "long", + "value": 0 + }, + "total_net_usage": { + "type": "long", + "value": 0 + }, + "urn": { + "type": "SessionID", + "value": "aff4:/hunts/H:556677" + } } }, "email_cc_addresses": [], @@ -3831,21 +5850,47 @@ "type": "RDFDatetime", "value": 2419244000000 }, - "id": { "type": "unicode", "value": "approval:333333" }, - "is_valid": { "type": "bool", "value": false }, + "id": { + "type": "unicode", + "value": "approval:333333" + }, + "is_valid": { + "type": "bool", + "value": false + }, "is_valid_message": { "type": "unicode", "value": "Need at least 1 additional approver for access." }, - "notified_users": [{ "type": "unicode", "value": "approver" }], - "reason": { "type": "unicode", "value": "foo" }, - "requestor": { "type": "unicode", "value": "api_test_user" }, + "notified_users": [ + { + "type": "unicode", + "value": "approver" + } + ], + "reason": { + "type": "unicode", + "value": "foo" + }, + "requestor": { + "type": "unicode", + "value": "api_test_user" + }, "subject": { "type": "ApiHunt", "value": { - "all_clients_count": { "type": "long", "value": 0 }, - "client_limit": { "type": "long", "value": 100 }, - "client_rate": { "type": "float", "value": 0.0 }, + "all_clients_count": { + "type": "long", + "value": 0 + }, + "client_limit": { + "type": "long", + "value": 100 + }, + "client_rate": { + "type": "float", + "value": 0.0 + }, "client_rule_set": { "type": "ForemanClientRuleSet", "value": { @@ -3875,29 +5920,68 @@ ] } }, - "clients_with_results_count": { "type": "long", "value": 0 }, - "completed_clients_count": { "type": "long", "value": 0 }, - "crash_limit": { "type": "long", "value": 100 }, - "crashed_clients_count": { "type": "long", "value": 0 }, - "created": { "type": "RDFDatetime", "value": 42000000 }, - "creator": { "type": "unicode", "value": "api_test_user" }, - "description": { "type": "unicode", "value": "copied hunt" }, - "duration": { "type": "DurationSeconds", "value": 1209600 }, - "failed_clients_count": { "type": "long", "value": 0 }, + "clients_with_results_count": { + "type": "long", + "value": 0 + }, + "completed_clients_count": { + "type": "long", + "value": 0 + }, + "crash_limit": { + "type": "long", + "value": 100 + }, + "crashed_clients_count": { + "type": "long", + "value": 0 + }, + "created": { + "type": "RDFDatetime", + "value": 42000000 + }, + "creator": { + "type": "unicode", + "value": "api_test_user" + }, + "description": { + "type": "unicode", + "value": "copied hunt" + }, + "duration": { + "type": "DurationSeconds", + "value": 1209600 + }, + "failed_clients_count": { + "type": "long", + "value": 0 + }, "flow_args": { "type": "GetFileArgs", "value": { "pathspec": { "type": "PathSpec", "value": { - "path": { "type": "unicode", "value": "/tmp/evil.txt" }, - "pathtype": { "type": "EnumNamedValue", "value": "OS" } + "path": { + "type": "unicode", + "value": "/tmp/evil.txt" + }, + "pathtype": { + "type": "EnumNamedValue", + "value": "OS" + } } } } }, - "flow_name": { "type": "unicode", "value": "GetFile" }, - "hunt_id": { "type": "ApiHuntId", "value": "H:DDEEFF" }, + "flow_name": { + "type": "unicode", + "value": "GetFile" + }, + "hunt_id": { + "type": "ApiHuntId", + "value": "H:DDEEFF" + }, "hunt_runner_args": { "type": "HuntRunnerArgs", "value": { @@ -3913,7 +5997,10 @@ "type": "long", "value": 1000 }, - "client_rate": { "type": "float", "value": 0.0 }, + "client_rate": { + "type": "float", + "value": 0.0 + }, "client_rule_set": { "type": "ForemanClientRuleSet", "value": { @@ -3943,20 +6030,32 @@ ] } }, - "crash_limit": { "type": "long", "value": 100 }, - "description": { "type": "unicode", "value": "copied hunt" }, + "crash_limit": { + "type": "long", + "value": 100 + }, + "description": { + "type": "unicode", + "value": "copied hunt" + }, "expiry_time": { "type": "DurationSeconds", "value": 1209600 }, - "hunt_name": { "type": "unicode", "value": "GenericHunt" }, + "hunt_name": { + "type": "unicode", + "value": "GenericHunt" + }, "original_object": { "type": "FlowLikeObjectReference", "value": { "hunt_reference": { "type": "HuntReference", "value": { - "hunt_id": { "type": "unicode", "value": "H:556677" } + "hunt_id": { + "type": "unicode", + "value": "H:556677" + } } }, "object_type": { @@ -3967,16 +6066,28 @@ } } }, - "hunt_type": { "type": "EnumNamedValue", "value": "STANDARD" }, - "is_robot": { "type": "bool", "value": false }, - "name": { "type": "unicode", "value": "GenericHunt" }, + "hunt_type": { + "type": "EnumNamedValue", + "value": "STANDARD" + }, + "is_robot": { + "type": "bool", + "value": false + }, + "name": { + "type": "unicode", + "value": "GenericHunt" + }, "original_object": { "type": "ApiFlowLikeObjectReference", "value": { "hunt_reference": { "type": "ApiHuntReference", "value": { - "hunt_id": { "type": "ApiHuntId", "value": "H:556677" } + "hunt_id": { + "type": "ApiHuntId", + "value": "H:556677" + } } }, "object_type": { @@ -3985,20 +6096,47 @@ } } }, - "remaining_clients_count": { "type": "long", "value": 0 }, - "results_count": { "type": "long", "value": 0 }, - "state": { "type": "EnumNamedValue", "value": "PAUSED" }, - "state_comment": { "type": "unicode", "value": "" }, - "total_cpu_usage": { "type": "long", "value": 0 }, - "total_net_usage": { "type": "long", "value": 0 }, - "urn": { "type": "SessionID", "value": "aff4:/hunts/H:DDEEFF" } + "remaining_clients_count": { + "type": "long", + "value": 0 + }, + "results_count": { + "type": "long", + "value": 0 + }, + "state": { + "type": "EnumNamedValue", + "value": "PAUSED" + }, + "state_comment": { + "type": "unicode", + "value": "" + }, + "state_reason": { + "type": "EnumNamedValue", + "value": "UNKNOWN" + }, + "total_cpu_usage": { + "type": "long", + "value": 0 + }, + "total_net_usage": { + "type": "long", + "value": 0 + }, + "urn": { + "type": "SessionID", + "value": "aff4:/hunts/H:DDEEFF" + } } } } }, "test_class": "ApiGetHuntApprovalHandlerRegressionTest_http_v1", "type_stripped_response": { - "approvers": ["api_test_user"], + "approvers": [ + "api_test_user" + ], "copied_from_hunt": { "all_clients_count": 0, "client_limit": 100, @@ -4006,7 +6144,10 @@ "client_rule_set": { "rules": [ { - "regex": { "attribute_regex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attribute_regex": "GRR", + "field": "CLIENT_NAME" + }, "rule_type": "REGEX" } ] @@ -4021,7 +6162,10 @@ "duration": 1209600, "failed_clients_count": 0, "flow_args": { - "pathspec": { "path": "/tmp/evil.txt", "pathtype": "OS" } + "pathspec": { + "path": "/tmp/evil.txt", + "pathtype": "OS" + } }, "flow_name": "GetFile", "hunt_id": "H:556677", @@ -4033,7 +6177,10 @@ "client_rule_set": { "rules": [ { - "regex": { "attribute_regex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attribute_regex": "GRR", + "field": "CLIENT_NAME" + }, "rule_type": "REGEX" } ] @@ -4051,6 +6198,7 @@ "results_count": 0, "state": "PAUSED", "state_comment": "", + "state_reason": "UNKNOWN", "total_cpu_usage": 0, "total_net_usage": 0, "urn": "aff4:/hunts/H:556677" @@ -4061,7 +6209,9 @@ "id": "approval:333333", "is_valid": false, "is_valid_message": "Need at least 1 additional approver for access.", - "notified_users": ["approver"], + "notified_users": [ + "approver" + ], "reason": "foo", "requestor": "api_test_user", "subject": { @@ -4071,7 +6221,10 @@ "client_rule_set": { "rules": [ { - "regex": { "attribute_regex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attribute_regex": "GRR", + "field": "CLIENT_NAME" + }, "rule_type": "REGEX" } ] @@ -4086,7 +6239,10 @@ "duration": 1209600, "failed_clients_count": 0, "flow_args": { - "pathspec": { "path": "/tmp/evil.txt", "pathtype": "OS" } + "pathspec": { + "path": "/tmp/evil.txt", + "pathtype": "OS" + } }, "flow_name": "GetFile", "hunt_id": "H:DDEEFF", @@ -4098,7 +6254,10 @@ "client_rule_set": { "rules": [ { - "regex": { "attribute_regex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attribute_regex": "GRR", + "field": "CLIENT_NAME" + }, "rule_type": "REGEX" } ] @@ -4108,7 +6267,9 @@ "expiry_time": 1209600, "hunt_name": "GenericHunt", "original_object": { - "hunt_reference": { "hunt_id": "H:556677" }, + "hunt_reference": { + "hunt_id": "H:556677" + }, "object_type": "HUNT_REFERENCE" } }, @@ -4116,13 +6277,16 @@ "is_robot": false, "name": "GenericHunt", "original_object": { - "hunt_reference": { "hunt_id": "H:556677" }, + "hunt_reference": { + "hunt_id": "H:556677" + }, "object_type": "HUNT_REFERENCE" }, "remaining_clients_count": 0, "results_count": 0, "state": "PAUSED", "state_comment": "", + "state_reason": "UNKNOWN", "total_cpu_usage": 0, "total_net_usage": 0, "urn": "aff4:/hunts/H:DDEEFF" @@ -4136,20 +6300,43 @@ "response": { "type": "ApiHuntApproval", "value": { - "approvers": [{ "type": "unicode", "value": "api_test_user" }], + "approvers": [ + { + "type": "unicode", + "value": "api_test_user" + } + ], "copied_from_flow": { "type": "ApiFlow", "value": { - "args": { "type": "InterrogateArgs", "value": {} }, + "args": { + "type": "InterrogateArgs", + "value": {} + }, "client_id": { "type": "ApiClientId", "value": "C.1000000000000000" }, - "creator": { "type": "unicode", "value": "api_test_user" }, - "flow_id": { "type": "ApiFlowId", "value": "F:112233" }, - "is_robot": { "type": "bool", "value": false }, - "last_active_at": { "type": "RDFDatetime", "value": 42000000 }, - "name": { "type": "unicode", "value": "Interrogate" }, + "creator": { + "type": "unicode", + "value": "api_test_user" + }, + "flow_id": { + "type": "ApiFlowId", + "value": "F:112233" + }, + "is_robot": { + "type": "bool", + "value": false + }, + "last_active_at": { + "type": "RDFDatetime", + "value": 42000000 + }, + "name": { + "type": "unicode", + "value": "Interrogate" + }, "runner_args": { "type": "FlowRunnerArgs", "value": { @@ -4157,12 +6344,24 @@ "type": "ClientURN", "value": "aff4:/C.1000000000000000" }, - "flow_name": { "type": "unicode", "value": "Interrogate" }, - "notify_to_user": { "type": "bool", "value": true } + "flow_name": { + "type": "unicode", + "value": "Interrogate" + }, + "notify_to_user": { + "type": "bool", + "value": true + } } }, - "started_at": { "type": "RDFDatetime", "value": 42000000 }, - "state": { "type": "EnumNamedValue", "value": "RUNNING" }, + "started_at": { + "type": "RDFDatetime", + "value": 42000000 + }, + "state": { + "type": "EnumNamedValue", + "value": "RUNNING" + }, "urn": { "type": "SessionID", "value": "aff4:/C.1000000000000000/flows/F:112233" @@ -4178,21 +6377,47 @@ "type": "RDFDatetime", "value": 2419244000000 }, - "id": { "type": "unicode", "value": "approval:444444" }, - "is_valid": { "type": "bool", "value": false }, + "id": { + "type": "unicode", + "value": "approval:444444" + }, + "is_valid": { + "type": "bool", + "value": false + }, "is_valid_message": { "type": "unicode", "value": "Need at least 1 additional approver for access." }, - "notified_users": [{ "type": "unicode", "value": "approver" }], - "reason": { "type": "unicode", "value": "foo" }, - "requestor": { "type": "unicode", "value": "api_test_user" }, + "notified_users": [ + { + "type": "unicode", + "value": "approver" + } + ], + "reason": { + "type": "unicode", + "value": "foo" + }, + "requestor": { + "type": "unicode", + "value": "api_test_user" + }, "subject": { "type": "ApiHunt", "value": { - "all_clients_count": { "type": "long", "value": 0 }, - "client_limit": { "type": "long", "value": 100 }, - "client_rate": { "type": "float", "value": 0.0 }, + "all_clients_count": { + "type": "long", + "value": 0 + }, + "client_limit": { + "type": "long", + "value": 100 + }, + "client_rate": { + "type": "float", + "value": 0.0 + }, "client_rule_set": { "type": "ForemanClientRuleSet", "value": { @@ -4222,32 +6447,68 @@ ] } }, - "clients_with_results_count": { "type": "long", "value": 0 }, - "completed_clients_count": { "type": "long", "value": 0 }, - "crash_limit": { "type": "long", "value": 100 }, - "crashed_clients_count": { "type": "long", "value": 0 }, - "created": { "type": "RDFDatetime", "value": 42000000 }, - "creator": { "type": "unicode", "value": "api_test_user" }, + "clients_with_results_count": { + "type": "long", + "value": 0 + }, + "completed_clients_count": { + "type": "long", + "value": 0 + }, + "crash_limit": { + "type": "long", + "value": 100 + }, + "crashed_clients_count": { + "type": "long", + "value": 0 + }, + "created": { + "type": "RDFDatetime", + "value": 42000000 + }, + "creator": { + "type": "unicode", + "value": "api_test_user" + }, "description": { "type": "unicode", "value": "hunt started from flow" }, - "duration": { "type": "DurationSeconds", "value": 1209600 }, - "failed_clients_count": { "type": "long", "value": 0 }, + "duration": { + "type": "DurationSeconds", + "value": 1209600 + }, + "failed_clients_count": { + "type": "long", + "value": 0 + }, "flow_args": { "type": "GetFileArgs", "value": { "pathspec": { "type": "PathSpec", "value": { - "path": { "type": "unicode", "value": "/tmp/evil.txt" }, - "pathtype": { "type": "EnumNamedValue", "value": "OS" } + "path": { + "type": "unicode", + "value": "/tmp/evil.txt" + }, + "pathtype": { + "type": "EnumNamedValue", + "value": "OS" + } } } } }, - "flow_name": { "type": "unicode", "value": "GetFile" }, - "hunt_id": { "type": "ApiHuntId", "value": "H:667788" }, + "flow_name": { + "type": "unicode", + "value": "GetFile" + }, + "hunt_id": { + "type": "ApiHuntId", + "value": "H:667788" + }, "hunt_runner_args": { "type": "HuntRunnerArgs", "value": { @@ -4263,7 +6524,10 @@ "type": "long", "value": 1000 }, - "client_rate": { "type": "float", "value": 0.0 }, + "client_rate": { + "type": "float", + "value": 0.0 + }, "client_rule_set": { "type": "ForemanClientRuleSet", "value": { @@ -4293,7 +6557,10 @@ ] } }, - "crash_limit": { "type": "long", "value": 100 }, + "crash_limit": { + "type": "long", + "value": 100 + }, "description": { "type": "unicode", "value": "hunt started from flow" @@ -4302,7 +6569,10 @@ "type": "DurationSeconds", "value": 1209600 }, - "hunt_name": { "type": "unicode", "value": "GenericHunt" }, + "hunt_name": { + "type": "unicode", + "value": "GenericHunt" + }, "original_object": { "type": "FlowLikeObjectReference", "value": { @@ -4313,7 +6583,10 @@ "type": "unicode", "value": "C.1000000000000000" }, - "flow_id": { "type": "unicode", "value": "F:112233" } + "flow_id": { + "type": "unicode", + "value": "F:112233" + } } }, "object_type": { @@ -4324,9 +6597,18 @@ } } }, - "hunt_type": { "type": "EnumNamedValue", "value": "STANDARD" }, - "is_robot": { "type": "bool", "value": false }, - "name": { "type": "unicode", "value": "GenericHunt" }, + "hunt_type": { + "type": "EnumNamedValue", + "value": "STANDARD" + }, + "is_robot": { + "type": "bool", + "value": false + }, + "name": { + "type": "unicode", + "value": "GenericHunt" + }, "original_object": { "type": "ApiFlowLikeObjectReference", "value": { @@ -4337,7 +6619,10 @@ "type": "ApiClientId", "value": "C.1000000000000000" }, - "flow_id": { "type": "ApiFlowId", "value": "F:112233" } + "flow_id": { + "type": "ApiFlowId", + "value": "F:112233" + } } }, "object_type": { @@ -4346,20 +6631,47 @@ } } }, - "remaining_clients_count": { "type": "long", "value": 0 }, - "results_count": { "type": "long", "value": 0 }, - "state": { "type": "EnumNamedValue", "value": "PAUSED" }, - "state_comment": { "type": "unicode", "value": "" }, - "total_cpu_usage": { "type": "long", "value": 0 }, - "total_net_usage": { "type": "long", "value": 0 }, - "urn": { "type": "SessionID", "value": "aff4:/hunts/H:667788" } + "remaining_clients_count": { + "type": "long", + "value": 0 + }, + "results_count": { + "type": "long", + "value": 0 + }, + "state": { + "type": "EnumNamedValue", + "value": "PAUSED" + }, + "state_comment": { + "type": "unicode", + "value": "" + }, + "state_reason": { + "type": "EnumNamedValue", + "value": "UNKNOWN" + }, + "total_cpu_usage": { + "type": "long", + "value": 0 + }, + "total_net_usage": { + "type": "long", + "value": 0 + }, + "urn": { + "type": "SessionID", + "value": "aff4:/hunts/H:667788" + } } } } }, "test_class": "ApiGetHuntApprovalHandlerRegressionTest_http_v1", "type_stripped_response": { - "approvers": ["api_test_user"], + "approvers": [ + "api_test_user" + ], "copied_from_flow": { "args": {}, "client_id": "C.1000000000000000", @@ -4383,7 +6695,9 @@ "id": "approval:444444", "is_valid": false, "is_valid_message": "Need at least 1 additional approver for access.", - "notified_users": ["approver"], + "notified_users": [ + "approver" + ], "reason": "foo", "requestor": "api_test_user", "subject": { @@ -4393,7 +6707,10 @@ "client_rule_set": { "rules": [ { - "regex": { "attribute_regex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attribute_regex": "GRR", + "field": "CLIENT_NAME" + }, "rule_type": "REGEX" } ] @@ -4408,7 +6725,10 @@ "duration": 1209600, "failed_clients_count": 0, "flow_args": { - "pathspec": { "path": "/tmp/evil.txt", "pathtype": "OS" } + "pathspec": { + "path": "/tmp/evil.txt", + "pathtype": "OS" + } }, "flow_name": "GetFile", "hunt_id": "H:667788", @@ -4420,7 +6740,10 @@ "client_rule_set": { "rules": [ { - "regex": { "attribute_regex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attribute_regex": "GRR", + "field": "CLIENT_NAME" + }, "rule_type": "REGEX" } ] @@ -4451,6 +6774,7 @@ "results_count": 0, "state": "PAUSED", "state_comment": "", + "state_reason": "UNKNOWN", "total_cpu_usage": 0, "total_net_usage": 0, "urn": "aff4:/hunts/H:667788" @@ -4465,30 +6789,96 @@ "method": "GET", "response": { "complete_points": [ - { "x_value": 44, "y_value": 0 }, - { "x_value": 45, "y_value": 1 }, - { "x_value": 55, "y_value": 2 }, - { "x_value": 65, "y_value": 3 }, - { "x_value": 75, "y_value": 4 }, - { "x_value": 85, "y_value": 5 }, - { "x_value": 95, "y_value": 6 }, - { "x_value": 105, "y_value": 7 }, - { "x_value": 115, "y_value": 8 }, - { "x_value": 125, "y_value": 9 }, - { "x_value": 135, "y_value": 10 } + { + "x_value": 44, + "y_value": 0 + }, + { + "x_value": 45, + "y_value": 1 + }, + { + "x_value": 55, + "y_value": 2 + }, + { + "x_value": 65, + "y_value": 3 + }, + { + "x_value": 75, + "y_value": 4 + }, + { + "x_value": 85, + "y_value": 5 + }, + { + "x_value": 95, + "y_value": 6 + }, + { + "x_value": 105, + "y_value": 7 + }, + { + "x_value": 115, + "y_value": 8 + }, + { + "x_value": 125, + "y_value": 9 + }, + { + "x_value": 135, + "y_value": 10 + } ], "start_points": [ - { "x_value": 44, "y_value": 0 }, - { "x_value": 45, "y_value": 1 }, - { "x_value": 55, "y_value": 2 }, - { "x_value": 65, "y_value": 3 }, - { "x_value": 75, "y_value": 4 }, - { "x_value": 85, "y_value": 5 }, - { "x_value": 95, "y_value": 6 }, - { "x_value": 105, "y_value": 7 }, - { "x_value": 115, "y_value": 8 }, - { "x_value": 125, "y_value": 9 }, - { "x_value": 135, "y_value": 10 } + { + "x_value": 44, + "y_value": 0 + }, + { + "x_value": 45, + "y_value": 1 + }, + { + "x_value": 55, + "y_value": 2 + }, + { + "x_value": 65, + "y_value": 3 + }, + { + "x_value": 75, + "y_value": 4 + }, + { + "x_value": 85, + "y_value": 5 + }, + { + "x_value": 95, + "y_value": 6 + }, + { + "x_value": 105, + "y_value": 7 + }, + { + "x_value": 115, + "y_value": 8 + }, + { + "x_value": 125, + "y_value": 9 + }, + { + "x_value": 135, + "y_value": 10 + } ] }, "test_class": "ApiGetHuntClientCompletionStatsHandlerRegressionTest_http_v1", @@ -4499,16 +6889,40 @@ "method": "GET", "response": { "complete_points": [ - { "x_value": 66.75, "y_value": 3 }, - { "x_value": 89.5, "y_value": 5 }, - { "x_value": 112.25, "y_value": 7 }, - { "x_value": 135.0, "y_value": 10 } + { + "x_value": 66.75, + "y_value": 3 + }, + { + "x_value": 89.5, + "y_value": 5 + }, + { + "x_value": 112.25, + "y_value": 7 + }, + { + "x_value": 135.0, + "y_value": 10 + } ], "start_points": [ - { "x_value": 66.75, "y_value": 3 }, - { "x_value": 89.5, "y_value": 5 }, - { "x_value": 112.25, "y_value": 7 }, - { "x_value": 135.0, "y_value": 10 } + { + "x_value": 66.75, + "y_value": 3 + }, + { + "x_value": 89.5, + "y_value": 5 + }, + { + "x_value": 112.25, + "y_value": 7 + }, + { + "x_value": 135.0, + "y_value": 10 + } ] }, "test_class": "ApiGetHuntClientCompletionStatsHandlerRegressionTest_http_v1", @@ -4519,30 +6933,96 @@ "method": "GET", "response": { "complete_points": [ - { "x_value": 44, "y_value": 0 }, - { "x_value": 45, "y_value": 1 }, - { "x_value": 55, "y_value": 2 }, - { "x_value": 65, "y_value": 3 }, - { "x_value": 75, "y_value": 4 }, - { "x_value": 85, "y_value": 5 }, - { "x_value": 95, "y_value": 6 }, - { "x_value": 105, "y_value": 7 }, - { "x_value": 115, "y_value": 8 }, - { "x_value": 125, "y_value": 9 }, - { "x_value": 135, "y_value": 10 } + { + "x_value": 44, + "y_value": 0 + }, + { + "x_value": 45, + "y_value": 1 + }, + { + "x_value": 55, + "y_value": 2 + }, + { + "x_value": 65, + "y_value": 3 + }, + { + "x_value": 75, + "y_value": 4 + }, + { + "x_value": 85, + "y_value": 5 + }, + { + "x_value": 95, + "y_value": 6 + }, + { + "x_value": 105, + "y_value": 7 + }, + { + "x_value": 115, + "y_value": 8 + }, + { + "x_value": 125, + "y_value": 9 + }, + { + "x_value": 135, + "y_value": 10 + } ], "start_points": [ - { "x_value": 44, "y_value": 0 }, - { "x_value": 45, "y_value": 1 }, - { "x_value": 55, "y_value": 2 }, - { "x_value": 65, "y_value": 3 }, - { "x_value": 75, "y_value": 4 }, - { "x_value": 85, "y_value": 5 }, - { "x_value": 95, "y_value": 6 }, - { "x_value": 105, "y_value": 7 }, - { "x_value": 115, "y_value": 8 }, - { "x_value": 125, "y_value": 9 }, - { "x_value": 135, "y_value": 10 } + { + "x_value": 44, + "y_value": 0 + }, + { + "x_value": 45, + "y_value": 1 + }, + { + "x_value": 55, + "y_value": 2 + }, + { + "x_value": 65, + "y_value": 3 + }, + { + "x_value": 75, + "y_value": 4 + }, + { + "x_value": 85, + "y_value": 5 + }, + { + "x_value": 95, + "y_value": 6 + }, + { + "x_value": 105, + "y_value": 7 + }, + { + "x_value": 115, + "y_value": 8 + }, + { + "x_value": 125, + "y_value": 9 + }, + { + "x_value": 135, + "y_value": 10 + } ] }, "test_class": "ApiGetHuntClientCompletionStatsHandlerRegressionTest_http_v1", @@ -4556,9 +7036,18 @@ "response": { "type": "ApiHunt", "value": { - "all_clients_count": { "type": "long", "value": 0 }, - "client_limit": { "type": "long", "value": 100 }, - "client_rate": { "type": "float", "value": 0.0 }, + "all_clients_count": { + "type": "long", + "value": 0 + }, + "client_limit": { + "type": "long", + "value": 100 + }, + "client_rate": { + "type": "float", + "value": 0.0 + }, "client_rule_set": { "type": "ForemanClientRuleSet", "value": { @@ -4579,35 +7068,77 @@ } } }, - "rule_type": { "type": "EnumNamedValue", "value": "REGEX" } + "rule_type": { + "type": "EnumNamedValue", + "value": "REGEX" + } } } ] } }, - "clients_with_results_count": { "type": "long", "value": 0 }, - "completed_clients_count": { "type": "long", "value": 0 }, - "crash_limit": { "type": "long", "value": 100 }, - "crashed_clients_count": { "type": "long", "value": 0 }, - "created": { "type": "RDFDatetime", "value": 42000000 }, - "creator": { "type": "unicode", "value": "api_test_user" }, - "description": { "type": "unicode", "value": "the hunt" }, - "duration": { "type": "DurationSeconds", "value": 1209600 }, - "failed_clients_count": { "type": "long", "value": 0 }, + "clients_with_results_count": { + "type": "long", + "value": 0 + }, + "completed_clients_count": { + "type": "long", + "value": 0 + }, + "crash_limit": { + "type": "long", + "value": 100 + }, + "crashed_clients_count": { + "type": "long", + "value": 0 + }, + "created": { + "type": "RDFDatetime", + "value": 42000000 + }, + "creator": { + "type": "unicode", + "value": "api_test_user" + }, + "description": { + "type": "unicode", + "value": "the hunt" + }, + "duration": { + "type": "DurationSeconds", + "value": 1209600 + }, + "failed_clients_count": { + "type": "long", + "value": 0 + }, "flow_args": { "type": "GetFileArgs", "value": { "pathspec": { "type": "PathSpec", "value": { - "path": { "type": "unicode", "value": "/tmp/evil.txt" }, - "pathtype": { "type": "EnumNamedValue", "value": "OS" } + "path": { + "type": "unicode", + "value": "/tmp/evil.txt" + }, + "pathtype": { + "type": "EnumNamedValue", + "value": "OS" + } } } } }, - "flow_name": { "type": "unicode", "value": "GetFile" }, - "hunt_id": { "type": "ApiHuntId", "value": "H:123456" }, + "flow_name": { + "type": "unicode", + "value": "GetFile" + }, + "hunt_id": { + "type": "ApiHuntId", + "value": "H:123456" + }, "hunt_runner_args": { "type": "HuntRunnerArgs", "value": { @@ -4619,8 +7150,14 @@ "type": "long", "value": 10485760 }, - "avg_results_per_client_limit": { "type": "long", "value": 1000 }, - "client_rate": { "type": "float", "value": 0.0 }, + "avg_results_per_client_limit": { + "type": "long", + "value": 1000 + }, + "client_rate": { + "type": "float", + "value": 0.0 + }, "client_rule_set": { "type": "ForemanClientRuleSet", "value": { @@ -4650,10 +7187,22 @@ ] } }, - "crash_limit": { "type": "long", "value": 100 }, - "description": { "type": "unicode", "value": "the hunt" }, - "expiry_time": { "type": "DurationSeconds", "value": 1209600 }, - "hunt_name": { "type": "unicode", "value": "GenericHunt" }, + "crash_limit": { + "type": "long", + "value": 100 + }, + "description": { + "type": "unicode", + "value": "the hunt" + }, + "expiry_time": { + "type": "DurationSeconds", + "value": 1209600 + }, + "hunt_name": { + "type": "unicode", + "value": "GenericHunt" + }, "original_object": { "type": "FlowLikeObjectReference", "value": { @@ -4664,7 +7213,10 @@ "type": "unicode", "value": "C.1111111111111111" }, - "flow_id": { "type": "unicode", "value": "F:332211" } + "flow_id": { + "type": "unicode", + "value": "F:332211" + } } }, "object_type": { @@ -4675,9 +7227,18 @@ } } }, - "hunt_type": { "type": "EnumNamedValue", "value": "STANDARD" }, - "is_robot": { "type": "bool", "value": false }, - "name": { "type": "unicode", "value": "GenericHunt" }, + "hunt_type": { + "type": "EnumNamedValue", + "value": "STANDARD" + }, + "is_robot": { + "type": "bool", + "value": false + }, + "name": { + "type": "unicode", + "value": "GenericHunt" + }, "original_object": { "type": "ApiFlowLikeObjectReference", "value": { @@ -4688,7 +7249,10 @@ "type": "ApiClientId", "value": "C.1111111111111111" }, - "flow_id": { "type": "ApiFlowId", "value": "F:332211" } + "flow_id": { + "type": "ApiFlowId", + "value": "F:332211" + } } }, "object_type": { @@ -4697,13 +7261,38 @@ } } }, - "remaining_clients_count": { "type": "long", "value": 0 }, - "results_count": { "type": "long", "value": 0 }, - "state": { "type": "EnumNamedValue", "value": "PAUSED" }, - "state_comment": { "type": "unicode", "value": "" }, - "total_cpu_usage": { "type": "long", "value": 0 }, - "total_net_usage": { "type": "long", "value": 0 }, - "urn": { "type": "SessionID", "value": "aff4:/hunts/H:123456" } + "remaining_clients_count": { + "type": "long", + "value": 0 + }, + "results_count": { + "type": "long", + "value": 0 + }, + "state": { + "type": "EnumNamedValue", + "value": "PAUSED" + }, + "state_comment": { + "type": "unicode", + "value": "" + }, + "state_reason": { + "type": "EnumNamedValue", + "value": "UNKNOWN" + }, + "total_cpu_usage": { + "type": "long", + "value": 0 + }, + "total_net_usage": { + "type": "long", + "value": 0 + }, + "urn": { + "type": "SessionID", + "value": "aff4:/hunts/H:123456" + } } }, "test_class": "ApiGetHuntHandlerFlowCopyRegressionTest_http_v1", @@ -4714,7 +7303,10 @@ "client_rule_set": { "rules": [ { - "regex": { "attribute_regex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attribute_regex": "GRR", + "field": "CLIENT_NAME" + }, "rule_type": "REGEX" } ] @@ -4729,7 +7321,10 @@ "duration": 1209600, "failed_clients_count": 0, "flow_args": { - "pathspec": { "path": "/tmp/evil.txt", "pathtype": "OS" } + "pathspec": { + "path": "/tmp/evil.txt", + "pathtype": "OS" + } }, "flow_name": "GetFile", "hunt_id": "H:123456", @@ -4741,7 +7336,10 @@ "client_rule_set": { "rules": [ { - "regex": { "attribute_regex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attribute_regex": "GRR", + "field": "CLIENT_NAME" + }, "rule_type": "REGEX" } ] @@ -4772,6 +7370,7 @@ "results_count": 0, "state": "PAUSED", "state_comment": "", + "state_reason": "UNKNOWN", "total_cpu_usage": 0, "total_net_usage": 0, "urn": "aff4:/hunts/H:123456" @@ -4784,9 +7383,18 @@ "response": { "type": "ApiHunt", "value": { - "all_clients_count": { "type": "long", "value": 0 }, - "client_limit": { "type": "long", "value": 100 }, - "client_rate": { "type": "float", "value": 0.0 }, + "all_clients_count": { + "type": "long", + "value": 0 + }, + "client_limit": { + "type": "long", + "value": 100 + }, + "client_rate": { + "type": "float", + "value": 0.0 + }, "client_rule_set": { "type": "ForemanClientRuleSet", "value": { @@ -4807,35 +7415,77 @@ } } }, - "rule_type": { "type": "EnumNamedValue", "value": "REGEX" } + "rule_type": { + "type": "EnumNamedValue", + "value": "REGEX" + } } } ] } }, - "clients_with_results_count": { "type": "long", "value": 0 }, - "completed_clients_count": { "type": "long", "value": 0 }, - "crash_limit": { "type": "long", "value": 100 }, - "crashed_clients_count": { "type": "long", "value": 0 }, - "created": { "type": "RDFDatetime", "value": 42000000 }, - "creator": { "type": "unicode", "value": "api_test_user" }, - "description": { "type": "unicode", "value": "the hunt" }, - "duration": { "type": "DurationSeconds", "value": 1209600 }, - "failed_clients_count": { "type": "long", "value": 0 }, + "clients_with_results_count": { + "type": "long", + "value": 0 + }, + "completed_clients_count": { + "type": "long", + "value": 0 + }, + "crash_limit": { + "type": "long", + "value": 100 + }, + "crashed_clients_count": { + "type": "long", + "value": 0 + }, + "created": { + "type": "RDFDatetime", + "value": 42000000 + }, + "creator": { + "type": "unicode", + "value": "api_test_user" + }, + "description": { + "type": "unicode", + "value": "the hunt" + }, + "duration": { + "type": "DurationSeconds", + "value": 1209600 + }, + "failed_clients_count": { + "type": "long", + "value": 0 + }, "flow_args": { "type": "GetFileArgs", "value": { "pathspec": { "type": "PathSpec", "value": { - "path": { "type": "unicode", "value": "/tmp/evil.txt" }, - "pathtype": { "type": "EnumNamedValue", "value": "OS" } + "path": { + "type": "unicode", + "value": "/tmp/evil.txt" + }, + "pathtype": { + "type": "EnumNamedValue", + "value": "OS" + } } } } }, - "flow_name": { "type": "unicode", "value": "GetFile" }, - "hunt_id": { "type": "ApiHuntId", "value": "H:123456" }, + "flow_name": { + "type": "unicode", + "value": "GetFile" + }, + "hunt_id": { + "type": "ApiHuntId", + "value": "H:123456" + }, "hunt_runner_args": { "type": "HuntRunnerArgs", "value": { @@ -4847,8 +7497,14 @@ "type": "long", "value": 10485760 }, - "avg_results_per_client_limit": { "type": "long", "value": 1000 }, - "client_rate": { "type": "float", "value": 0.0 }, + "avg_results_per_client_limit": { + "type": "long", + "value": 1000 + }, + "client_rate": { + "type": "float", + "value": 0.0 + }, "client_rule_set": { "type": "ForemanClientRuleSet", "value": { @@ -4878,17 +7534,32 @@ ] } }, - "crash_limit": { "type": "long", "value": 100 }, - "description": { "type": "unicode", "value": "the hunt" }, - "expiry_time": { "type": "DurationSeconds", "value": 1209600 }, - "hunt_name": { "type": "unicode", "value": "GenericHunt" }, + "crash_limit": { + "type": "long", + "value": 100 + }, + "description": { + "type": "unicode", + "value": "the hunt" + }, + "expiry_time": { + "type": "DurationSeconds", + "value": 1209600 + }, + "hunt_name": { + "type": "unicode", + "value": "GenericHunt" + }, "original_object": { "type": "FlowLikeObjectReference", "value": { "hunt_reference": { "type": "HuntReference", "value": { - "hunt_id": { "type": "unicode", "value": "H:332211" } + "hunt_id": { + "type": "unicode", + "value": "H:332211" + } } }, "object_type": { @@ -4899,16 +7570,28 @@ } } }, - "hunt_type": { "type": "EnumNamedValue", "value": "STANDARD" }, - "is_robot": { "type": "bool", "value": false }, - "name": { "type": "unicode", "value": "GenericHunt" }, + "hunt_type": { + "type": "EnumNamedValue", + "value": "STANDARD" + }, + "is_robot": { + "type": "bool", + "value": false + }, + "name": { + "type": "unicode", + "value": "GenericHunt" + }, "original_object": { "type": "ApiFlowLikeObjectReference", "value": { "hunt_reference": { "type": "ApiHuntReference", "value": { - "hunt_id": { "type": "ApiHuntId", "value": "H:332211" } + "hunt_id": { + "type": "ApiHuntId", + "value": "H:332211" + } } }, "object_type": { @@ -4917,24 +7600,52 @@ } } }, - "remaining_clients_count": { "type": "long", "value": 0 }, - "results_count": { "type": "long", "value": 0 }, - "state": { "type": "EnumNamedValue", "value": "PAUSED" }, - "state_comment": { "type": "unicode", "value": "" }, - "total_cpu_usage": { "type": "long", "value": 0 }, - "total_net_usage": { "type": "long", "value": 0 }, - "urn": { "type": "SessionID", "value": "aff4:/hunts/H:123456" } - } - }, - "test_class": "ApiGetHuntHandlerHuntCopyRegressionTest_http_v1", - "type_stripped_response": { + "remaining_clients_count": { + "type": "long", + "value": 0 + }, + "results_count": { + "type": "long", + "value": 0 + }, + "state": { + "type": "EnumNamedValue", + "value": "PAUSED" + }, + "state_comment": { + "type": "unicode", + "value": "" + }, + "state_reason": { + "type": "EnumNamedValue", + "value": "UNKNOWN" + }, + "total_cpu_usage": { + "type": "long", + "value": 0 + }, + "total_net_usage": { + "type": "long", + "value": 0 + }, + "urn": { + "type": "SessionID", + "value": "aff4:/hunts/H:123456" + } + } + }, + "test_class": "ApiGetHuntHandlerHuntCopyRegressionTest_http_v1", + "type_stripped_response": { "all_clients_count": 0, "client_limit": 100, "client_rate": 0.0, "client_rule_set": { "rules": [ { - "regex": { "attribute_regex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attribute_regex": "GRR", + "field": "CLIENT_NAME" + }, "rule_type": "REGEX" } ] @@ -4949,7 +7660,10 @@ "duration": 1209600, "failed_clients_count": 0, "flow_args": { - "pathspec": { "path": "/tmp/evil.txt", "pathtype": "OS" } + "pathspec": { + "path": "/tmp/evil.txt", + "pathtype": "OS" + } }, "flow_name": "GetFile", "hunt_id": "H:123456", @@ -4961,7 +7675,10 @@ "client_rule_set": { "rules": [ { - "regex": { "attribute_regex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attribute_regex": "GRR", + "field": "CLIENT_NAME" + }, "rule_type": "REGEX" } ] @@ -4971,7 +7688,9 @@ "expiry_time": 1209600, "hunt_name": "GenericHunt", "original_object": { - "hunt_reference": { "hunt_id": "H:332211" }, + "hunt_reference": { + "hunt_id": "H:332211" + }, "object_type": "HUNT_REFERENCE" } }, @@ -4979,13 +7698,16 @@ "is_robot": false, "name": "GenericHunt", "original_object": { - "hunt_reference": { "hunt_id": "H:332211" }, + "hunt_reference": { + "hunt_id": "H:332211" + }, "object_type": "HUNT_REFERENCE" }, "remaining_clients_count": 0, "results_count": 0, "state": "PAUSED", "state_comment": "", + "state_reason": "UNKNOWN", "total_cpu_usage": 0, "total_net_usage": 0, "urn": "aff4:/hunts/H:123456" @@ -4998,9 +7720,18 @@ "response": { "type": "ApiHunt", "value": { - "all_clients_count": { "type": "long", "value": 0 }, - "client_limit": { "type": "long", "value": 100 }, - "client_rate": { "type": "float", "value": 0.0 }, + "all_clients_count": { + "type": "long", + "value": 0 + }, + "client_limit": { + "type": "long", + "value": 100 + }, + "client_rate": { + "type": "float", + "value": 0.0 + }, "client_rule_set": { "type": "ForemanClientRuleSet", "value": { @@ -5021,35 +7752,77 @@ } } }, - "rule_type": { "type": "EnumNamedValue", "value": "REGEX" } + "rule_type": { + "type": "EnumNamedValue", + "value": "REGEX" + } } } ] } }, - "clients_with_results_count": { "type": "long", "value": 0 }, - "completed_clients_count": { "type": "long", "value": 0 }, - "crash_limit": { "type": "long", "value": 100 }, - "crashed_clients_count": { "type": "long", "value": 0 }, - "created": { "type": "RDFDatetime", "value": 42000000 }, - "creator": { "type": "unicode", "value": "api_test_user" }, - "description": { "type": "unicode", "value": "the hunt" }, - "duration": { "type": "DurationSeconds", "value": 1209600 }, - "failed_clients_count": { "type": "long", "value": 0 }, + "clients_with_results_count": { + "type": "long", + "value": 0 + }, + "completed_clients_count": { + "type": "long", + "value": 0 + }, + "crash_limit": { + "type": "long", + "value": 100 + }, + "crashed_clients_count": { + "type": "long", + "value": 0 + }, + "created": { + "type": "RDFDatetime", + "value": 42000000 + }, + "creator": { + "type": "unicode", + "value": "api_test_user" + }, + "description": { + "type": "unicode", + "value": "the hunt" + }, + "duration": { + "type": "DurationSeconds", + "value": 1209600 + }, + "failed_clients_count": { + "type": "long", + "value": 0 + }, "flow_args": { "type": "GetFileArgs", "value": { "pathspec": { "type": "PathSpec", "value": { - "path": { "type": "unicode", "value": "/tmp/evil.txt" }, - "pathtype": { "type": "EnumNamedValue", "value": "OS" } + "path": { + "type": "unicode", + "value": "/tmp/evil.txt" + }, + "pathtype": { + "type": "EnumNamedValue", + "value": "OS" + } } } } }, - "flow_name": { "type": "unicode", "value": "GetFile" }, - "hunt_id": { "type": "ApiHuntId", "value": "H:123456" }, + "flow_name": { + "type": "unicode", + "value": "GetFile" + }, + "hunt_id": { + "type": "ApiHuntId", + "value": "H:123456" + }, "hunt_runner_args": { "type": "HuntRunnerArgs", "value": { @@ -5061,8 +7834,14 @@ "type": "long", "value": 10485760 }, - "avg_results_per_client_limit": { "type": "long", "value": 1000 }, - "client_rate": { "type": "float", "value": 0.0 }, + "avg_results_per_client_limit": { + "type": "long", + "value": 1000 + }, + "client_rate": { + "type": "float", + "value": 0.0 + }, "client_rule_set": { "type": "ForemanClientRuleSet", "value": { @@ -5092,26 +7871,72 @@ ] } }, - "crash_limit": { "type": "long", "value": 100 }, - "description": { "type": "unicode", "value": "the hunt" }, - "expiry_time": { "type": "DurationSeconds", "value": 1209600 }, - "hunt_name": { "type": "unicode", "value": "GenericHunt" }, + "crash_limit": { + "type": "long", + "value": 100 + }, + "description": { + "type": "unicode", + "value": "the hunt" + }, + "expiry_time": { + "type": "DurationSeconds", + "value": 1209600 + }, + "hunt_name": { + "type": "unicode", + "value": "GenericHunt" + }, "original_object": { "type": "FlowLikeObjectReference", "value": {} } } }, - "hunt_type": { "type": "EnumNamedValue", "value": "STANDARD" }, - "is_robot": { "type": "bool", "value": false }, - "name": { "type": "unicode", "value": "GenericHunt" }, - "remaining_clients_count": { "type": "long", "value": 0 }, - "results_count": { "type": "long", "value": 0 }, - "state": { "type": "EnumNamedValue", "value": "PAUSED" }, - "state_comment": { "type": "unicode", "value": "" }, - "total_cpu_usage": { "type": "long", "value": 0 }, - "total_net_usage": { "type": "long", "value": 0 }, - "urn": { "type": "SessionID", "value": "aff4:/hunts/H:123456" } + "hunt_type": { + "type": "EnumNamedValue", + "value": "STANDARD" + }, + "is_robot": { + "type": "bool", + "value": false + }, + "name": { + "type": "unicode", + "value": "GenericHunt" + }, + "remaining_clients_count": { + "type": "long", + "value": 0 + }, + "results_count": { + "type": "long", + "value": 0 + }, + "state": { + "type": "EnumNamedValue", + "value": "PAUSED" + }, + "state_comment": { + "type": "unicode", + "value": "" + }, + "state_reason": { + "type": "EnumNamedValue", + "value": "UNKNOWN" + }, + "total_cpu_usage": { + "type": "long", + "value": 0 + }, + "total_net_usage": { + "type": "long", + "value": 0 + }, + "urn": { + "type": "SessionID", + "value": "aff4:/hunts/H:123456" + } } }, "test_class": "ApiGetHuntHandlerRegressionTest_http_v1", @@ -5122,7 +7947,10 @@ "client_rule_set": { "rules": [ { - "regex": { "attribute_regex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attribute_regex": "GRR", + "field": "CLIENT_NAME" + }, "rule_type": "REGEX" } ] @@ -5137,7 +7965,10 @@ "duration": 1209600, "failed_clients_count": 0, "flow_args": { - "pathspec": { "path": "/tmp/evil.txt", "pathtype": "OS" } + "pathspec": { + "path": "/tmp/evil.txt", + "pathtype": "OS" + } }, "flow_name": "GetFile", "hunt_id": "H:123456", @@ -5149,7 +7980,10 @@ "client_rule_set": { "rules": [ { - "regex": { "attribute_regex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attribute_regex": "GRR", + "field": "CLIENT_NAME" + }, "rule_type": "REGEX" } ] @@ -5167,6 +8001,7 @@ "results_count": 0, "state": "PAUSED", "state_comment": "", + "state_reason": "UNKNOWN", "total_cpu_usage": 0, "total_net_usage": 0, "urn": "aff4:/hunts/H:123456" @@ -5203,38 +8038,59 @@ { "type": "StatsHistogramBin", "value": { - "num": { "type": "long", "value": 1 }, - "range_max_value": { "type": "float", "value": 16.0 } + "num": { + "type": "long", + "value": 1 + }, + "range_max_value": { + "type": "float", + "value": 16.0 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 32.0 } + "range_max_value": { + "type": "float", + "value": 32.0 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 64.0 } + "range_max_value": { + "type": "float", + "value": 64.0 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 128.0 } + "range_max_value": { + "type": "float", + "value": 128.0 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 256.0 } + "range_max_value": { + "type": "float", + "value": 256.0 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 512.0 } + "range_max_value": { + "type": "float", + "value": 512.0 + } } }, { @@ -5348,9 +8204,18 @@ ] } }, - "num": { "type": "long", "value": 1 }, - "stddev": { "type": "float", "value": 0.0 }, - "sum": { "type": "float", "value": 3.0 } + "num": { + "type": "long", + "value": 1 + }, + "stddev": { + "type": "float", + "value": 0.0 + }, + "sum": { + "type": "float", + "value": 3.0 + } } }, "system_cpu_stats": { @@ -5399,106 +8264,166 @@ { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 0.5 } + "range_max_value": { + "type": "float", + "value": 0.5 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 0.75 } + "range_max_value": { + "type": "float", + "value": 0.75 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 1.0 } + "range_max_value": { + "type": "float", + "value": 1.0 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 1.5 } + "range_max_value": { + "type": "float", + "value": 1.5 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 2.0 } + "range_max_value": { + "type": "float", + "value": 2.0 + } } }, { "type": "StatsHistogramBin", "value": { - "num": { "type": "long", "value": 1 }, - "range_max_value": { "type": "float", "value": 2.5 } + "num": { + "type": "long", + "value": 1 + }, + "range_max_value": { + "type": "float", + "value": 2.5 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 3.0 } + "range_max_value": { + "type": "float", + "value": 3.0 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 4.0 } + "range_max_value": { + "type": "float", + "value": 4.0 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 5.0 } + "range_max_value": { + "type": "float", + "value": 5.0 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 6.0 } + "range_max_value": { + "type": "float", + "value": 6.0 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 7.0 } + "range_max_value": { + "type": "float", + "value": 7.0 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 8.0 } + "range_max_value": { + "type": "float", + "value": 8.0 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 9.0 } + "range_max_value": { + "type": "float", + "value": 9.0 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 10.0 } + "range_max_value": { + "type": "float", + "value": 10.0 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 15.0 } + "range_max_value": { + "type": "float", + "value": 15.0 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 20.0 } + "range_max_value": { + "type": "float", + "value": 20.0 + } } } ] } }, - "num": { "type": "long", "value": 1 }, - "stddev": { "type": "float", "value": 0.0 }, - "sum": { "type": "float", "value": 2.0 } + "num": { + "type": "long", + "value": 1 + }, + "stddev": { + "type": "float", + "value": 0.0 + }, + "sum": { + "type": "float", + "value": 2.0 + } } }, "user_cpu_stats": { @@ -5547,106 +8472,166 @@ { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 0.5 } + "range_max_value": { + "type": "float", + "value": 0.5 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 0.75 } + "range_max_value": { + "type": "float", + "value": 0.75 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 1.0 } + "range_max_value": { + "type": "float", + "value": 1.0 + } } }, { "type": "StatsHistogramBin", "value": { - "num": { "type": "long", "value": 1 }, - "range_max_value": { "type": "float", "value": 1.5 } + "num": { + "type": "long", + "value": 1 + }, + "range_max_value": { + "type": "float", + "value": 1.5 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 2.0 } + "range_max_value": { + "type": "float", + "value": 2.0 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 2.5 } + "range_max_value": { + "type": "float", + "value": 2.5 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 3.0 } + "range_max_value": { + "type": "float", + "value": 3.0 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 4.0 } + "range_max_value": { + "type": "float", + "value": 4.0 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 5.0 } + "range_max_value": { + "type": "float", + "value": 5.0 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 6.0 } + "range_max_value": { + "type": "float", + "value": 6.0 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 7.0 } + "range_max_value": { + "type": "float", + "value": 7.0 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 8.0 } + "range_max_value": { + "type": "float", + "value": 8.0 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 9.0 } + "range_max_value": { + "type": "float", + "value": 9.0 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 10.0 } + "range_max_value": { + "type": "float", + "value": 10.0 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 15.0 } + "range_max_value": { + "type": "float", + "value": 15.0 + } } }, { "type": "StatsHistogramBin", "value": { - "range_max_value": { "type": "float", "value": 20.0 } + "range_max_value": { + "type": "float", + "value": 20.0 + } } } ] } }, - "num": { "type": "long", "value": 1 }, - "stddev": { "type": "float", "value": 0.0 }, - "sum": { "type": "float", "value": 1.0 } + "num": { + "type": "long", + "value": 1 + }, + "stddev": { + "type": "float", + "value": 0.0 + }, + "sum": { + "type": "float", + "value": 1.0 + } } }, "worst_performers": [ @@ -5660,11 +8645,20 @@ "cpu_usage": { "type": "CpuSeconds", "value": { - "system_cpu_time": { "type": "float", "value": 2.0 }, - "user_cpu_time": { "type": "float", "value": 1.0 } + "system_cpu_time": { + "type": "float", + "value": 2.0 + }, + "user_cpu_time": { + "type": "float", + "value": 1.0 + } } }, - "network_bytes_sent": { "type": "long", "value": 3 }, + "network_bytes_sent": { + "type": "long", + "value": 3 + }, "session_id": { "type": "SessionID", "value": "" @@ -5681,24 +8675,61 @@ "network_bytes_sent_stats": { "histogram": { "bins": [ - { "num": 1, "range_max_value": 16.0 }, - { "range_max_value": 32.0 }, - { "range_max_value": 64.0 }, - { "range_max_value": 128.0 }, - { "range_max_value": 256.0 }, - { "range_max_value": 512.0 }, - { "range_max_value": 1024.0 }, - { "range_max_value": 2048.0 }, - { "range_max_value": 4096.0 }, - { "range_max_value": 8192.0 }, - { "range_max_value": 16384.0 }, - { "range_max_value": 32768.0 }, - { "range_max_value": 65536.0 }, - { "range_max_value": 131072.0 }, - { "range_max_value": 262144.0 }, - { "range_max_value": 524288.0 }, - { "range_max_value": 1048576.0 }, - { "range_max_value": 2097152.0 } + { + "num": 1, + "range_max_value": 16.0 + }, + { + "range_max_value": 32.0 + }, + { + "range_max_value": 64.0 + }, + { + "range_max_value": 128.0 + }, + { + "range_max_value": 256.0 + }, + { + "range_max_value": 512.0 + }, + { + "range_max_value": 1024.0 + }, + { + "range_max_value": 2048.0 + }, + { + "range_max_value": 4096.0 + }, + { + "range_max_value": 8192.0 + }, + { + "range_max_value": 16384.0 + }, + { + "range_max_value": 32768.0 + }, + { + "range_max_value": 65536.0 + }, + { + "range_max_value": 131072.0 + }, + { + "range_max_value": 262144.0 + }, + { + "range_max_value": 524288.0 + }, + { + "range_max_value": 1048576.0 + }, + { + "range_max_value": 2097152.0 + } ] }, "num": 1, @@ -5708,55 +8739,137 @@ "system_cpu_stats": { "histogram": { "bins": [ - { "range_max_value": 0.10000000149011612 }, - { "range_max_value": 0.20000000298023224 }, - { "range_max_value": 0.30000001192092896 }, - { "range_max_value": 0.4000000059604645 }, - { "range_max_value": 0.5 }, - { "range_max_value": 0.75 }, - { "range_max_value": 1.0 }, - { "range_max_value": 1.5 }, - { "range_max_value": 2.0 }, - { "num": 1, "range_max_value": 2.5 }, - { "range_max_value": 3.0 }, - { "range_max_value": 4.0 }, - { "range_max_value": 5.0 }, - { "range_max_value": 6.0 }, - { "range_max_value": 7.0 }, - { "range_max_value": 8.0 }, - { "range_max_value": 9.0 }, - { "range_max_value": 10.0 }, - { "range_max_value": 15.0 }, - { "range_max_value": 20.0 } - ] - }, - "num": 1, - "stddev": 0.0, - "sum": 2.0 - }, - "user_cpu_stats": { - "histogram": { + { + "range_max_value": 0.10000000149011612 + }, + { + "range_max_value": 0.20000000298023224 + }, + { + "range_max_value": 0.30000001192092896 + }, + { + "range_max_value": 0.4000000059604645 + }, + { + "range_max_value": 0.5 + }, + { + "range_max_value": 0.75 + }, + { + "range_max_value": 1.0 + }, + { + "range_max_value": 1.5 + }, + { + "range_max_value": 2.0 + }, + { + "num": 1, + "range_max_value": 2.5 + }, + { + "range_max_value": 3.0 + }, + { + "range_max_value": 4.0 + }, + { + "range_max_value": 5.0 + }, + { + "range_max_value": 6.0 + }, + { + "range_max_value": 7.0 + }, + { + "range_max_value": 8.0 + }, + { + "range_max_value": 9.0 + }, + { + "range_max_value": 10.0 + }, + { + "range_max_value": 15.0 + }, + { + "range_max_value": 20.0 + } + ] + }, + "num": 1, + "stddev": 0.0, + "sum": 2.0 + }, + "user_cpu_stats": { + "histogram": { "bins": [ - { "range_max_value": 0.10000000149011612 }, - { "range_max_value": 0.20000000298023224 }, - { "range_max_value": 0.30000001192092896 }, - { "range_max_value": 0.4000000059604645 }, - { "range_max_value": 0.5 }, - { "range_max_value": 0.75 }, - { "range_max_value": 1.0 }, - { "num": 1, "range_max_value": 1.5 }, - { "range_max_value": 2.0 }, - { "range_max_value": 2.5 }, - { "range_max_value": 3.0 }, - { "range_max_value": 4.0 }, - { "range_max_value": 5.0 }, - { "range_max_value": 6.0 }, - { "range_max_value": 7.0 }, - { "range_max_value": 8.0 }, - { "range_max_value": 9.0 }, - { "range_max_value": 10.0 }, - { "range_max_value": 15.0 }, - { "range_max_value": 20.0 } + { + "range_max_value": 0.10000000149011612 + }, + { + "range_max_value": 0.20000000298023224 + }, + { + "range_max_value": 0.30000001192092896 + }, + { + "range_max_value": 0.4000000059604645 + }, + { + "range_max_value": 0.5 + }, + { + "range_max_value": 0.75 + }, + { + "range_max_value": 1.0 + }, + { + "num": 1, + "range_max_value": 1.5 + }, + { + "range_max_value": 2.0 + }, + { + "range_max_value": 2.5 + }, + { + "range_max_value": 3.0 + }, + { + "range_max_value": 4.0 + }, + { + "range_max_value": 5.0 + }, + { + "range_max_value": 6.0 + }, + { + "range_max_value": 7.0 + }, + { + "range_max_value": 8.0 + }, + { + "range_max_value": 9.0 + }, + { + "range_max_value": 10.0 + }, + { + "range_max_value": 15.0 + }, + { + "range_max_value": 20.0 + } ] }, "num": 1, @@ -5766,7 +8879,10 @@ "worst_performers": [ { "client_id": "aff4:/C.1000000000000000", - "cpu_usage": { "system_cpu_time": 2.0, "user_cpu_time": 1.0 }, + "cpu_usage": { + "system_cpu_time": 2.0, + "user_cpu_time": 1.0 + }, "network_bytes_sent": 3, "session_id": "" } @@ -5781,9 +8897,9 @@ "api_method": "GetLastClientIPAddress", "method": "GET", "response": { - "info": "Internal IP address.", - "ip": "192.168.100.42", - "status": "INTERNAL" + "info": "No ip information.", + "ip": "", + "status": "UNKNOWN" }, "test_class": "ApiGetLastClientIPAddressHandlerRegressionTest_http_v1", "url": "/api/clients/C.1000000000000000/last-ip" @@ -5811,9 +8927,18 @@ "type": "bool", "value": true }, - "create_hunt_action_enabled": { "type": "bool", "value": true }, - "cron_jobs_nav_item_enabled": { "type": "bool", "value": true }, - "hunt_approval_required": { "type": "bool", "value": false }, + "create_hunt_action_enabled": { + "type": "bool", + "value": true + }, + "cron_jobs_nav_item_enabled": { + "type": "bool", + "value": true + }, + "hunt_approval_required": { + "type": "bool", + "value": false + }, "hunt_manager_nav_item_enabled": { "type": "bool", "value": true @@ -5834,8 +8959,14 @@ "type": "bool", "value": true }, - "server_load_nav_item_enabled": { "type": "bool", "value": true }, - "settings_nav_item_enabled": { "type": "bool", "value": true }, + "server_load_nav_item_enabled": { + "type": "bool", + "value": true + }, + "settings_nav_item_enabled": { + "type": "bool", + "value": true + }, "show_statistics_nav_item_enabled": { "type": "bool", "value": true @@ -5848,21 +8979,33 @@ "type": "bool", "value": true }, - "upload_binary_action_enabled": { "type": "bool", "value": true } + "upload_binary_action_enabled": { + "type": "bool", + "value": true + } } }, "settings": { "type": "GUISettings", "value": { - "canary_mode": { "type": "bool", "value": true }, - "mode": { "type": "EnumNamedValue", "value": "ADVANCED" } + "canary_mode": { + "type": "bool", + "value": true + }, + "mode": { + "type": "EnumNamedValue", + "value": "ADVANCED" + } } }, "user_type": { "type": "EnumNamedValue", "value": "USER_TYPE_STANDARD" }, - "username": { "type": "unicode", "value": "api_test_user" } + "username": { + "type": "unicode", + "value": "api_test_user" + } } }, "test_class": "ApiGetOwnGrrUserHandlerRegressionTest_http_v1", @@ -5886,7 +9029,10 @@ "upload_artifact_action_enabled": true, "upload_binary_action_enabled": true }, - "settings": { "canary_mode": true, "mode": "ADVANCED" }, + "settings": { + "canary_mode": true, + "mode": "ADVANCED" + }, "user_type": "USER_TYPE_STANDARD", "username": "api_test_user" }, @@ -5913,9 +9059,18 @@ "type": "bool", "value": true }, - "create_hunt_action_enabled": { "type": "bool", "value": true }, - "cron_jobs_nav_item_enabled": { "type": "bool", "value": true }, - "hunt_approval_required": { "type": "bool", "value": false }, + "create_hunt_action_enabled": { + "type": "bool", + "value": true + }, + "cron_jobs_nav_item_enabled": { + "type": "bool", + "value": true + }, + "hunt_approval_required": { + "type": "bool", + "value": false + }, "hunt_manager_nav_item_enabled": { "type": "bool", "value": true @@ -5936,8 +9091,14 @@ "type": "bool", "value": true }, - "server_load_nav_item_enabled": { "type": "bool", "value": true }, - "settings_nav_item_enabled": { "type": "bool", "value": true }, + "server_load_nav_item_enabled": { + "type": "bool", + "value": true + }, + "settings_nav_item_enabled": { + "type": "bool", + "value": true + }, "show_statistics_nav_item_enabled": { "type": "bool", "value": true @@ -5950,18 +9111,33 @@ "type": "bool", "value": true }, - "upload_binary_action_enabled": { "type": "bool", "value": true } + "upload_binary_action_enabled": { + "type": "bool", + "value": true + } } }, "settings": { "type": "GUISettings", "value": { - "canary_mode": { "type": "bool", "value": true }, - "mode": { "type": "EnumNamedValue", "value": "ADVANCED" } + "canary_mode": { + "type": "bool", + "value": true + }, + "mode": { + "type": "EnumNamedValue", + "value": "ADVANCED" + } } }, - "user_type": { "type": "EnumNamedValue", "value": "USER_TYPE_ADMIN" }, - "username": { "type": "unicode", "value": "api_test_user" } + "user_type": { + "type": "EnumNamedValue", + "value": "USER_TYPE_ADMIN" + }, + "username": { + "type": "unicode", + "value": "api_test_user" + } } }, "test_class": "ApiGetOwnGrrUserHandlerRegressionTest_http_v1", @@ -5985,7 +9161,10 @@ "upload_artifact_action_enabled": true, "upload_binary_action_enabled": true }, - "settings": { "canary_mode": true, "mode": "ADVANCED" }, + "settings": { + "canary_mode": true, + "mode": "ADVANCED" + }, "user_type": "USER_TYPE_ADMIN", "username": "api_test_user" }, @@ -5996,7 +9175,9 @@ { "api_method": "GetPendingUserNotificationsCount", "method": "GET", - "response": { "count": 2 }, + "response": { + "count": 2 + }, "test_class": "ApiGetPendingUserNotificationsCountHandlerRegressionTest_http_v1", "url": "/api/users/me/notifications/pending/count" } @@ -6006,7 +9187,10 @@ "api_method": "GetRDFValueDescriptor", "method": "GET", "response": { - "default": { "type": "DurationSeconds", "value": 0 }, + "default": { + "type": "DurationSeconds", + "value": 0 + }, "doc": "Duration that is (de)serialized with second-precision.\n\n This class exists for compatibility purposes and to keep certain API fields\n simple. For most uses, please prefer `Duration` directly.\n ", "kind": "primitive", "mro": [ @@ -6038,7 +9222,10 @@ "api_method": "GetRDFValueDescriptor", "method": "GET", "response": { - "default": { "type": "ApiFlow", "value": {} }, + "default": { + "type": "ApiFlow", + "value": {} + }, "doc": "ApiFlow is used when rendering responses.\n\n ApiFlow is meant to be more lightweight than automatically generated AFF4\n representation. It's also meant to contain only the information needed by\n the UI and and to not expose implementation defails.\n ", "fields": [ { @@ -6069,7 +9256,10 @@ "type": "ApiClientId" }, { - "default": { "type": "RDFString", "value": "" }, + "default": { + "type": "RDFString", + "value": "" + }, "doc": "Flow name.", "dynamic": false, "friendly_name": "Name", @@ -6114,12 +9304,35 @@ }, { "allowed_values": [ - { "doc": "", "labels": [], "name": "RUNNING", "value": 0 }, - { "doc": "", "labels": [], "name": "TERMINATED", "value": 1 }, - { "doc": "", "labels": [], "name": "ERROR", "value": 3 }, - { "doc": "", "labels": [], "name": "CLIENT_CRASHED", "value": 4 } + { + "doc": "", + "labels": [], + "name": "RUNNING", + "value": 0 + }, + { + "doc": "", + "labels": [], + "name": "TERMINATED", + "value": 1 + }, + { + "doc": "", + "labels": [], + "name": "ERROR", + "value": 3 + }, + { + "doc": "", + "labels": [], + "name": "CLIENT_CRASHED", + "value": 4 + } ], - "default": { "type": "EnumNamedValue", "value": "RUNNING" }, + "default": { + "type": "EnumNamedValue", + "value": "RUNNING" + }, "doc": "Current flow state.", "dynamic": false, "friendly_name": "State", @@ -6129,7 +9342,10 @@ "type": "EnumNamedValue" }, { - "default": { "type": "RDFString", "value": "" }, + "default": { + "type": "RDFString", + "value": "" + }, "doc": "Describes the error that happened while executing the flow, truncated if over 4 KB.", "dynamic": false, "friendly_name": "Error description", @@ -6157,7 +9373,10 @@ "type": "RDFDatetime" }, { - "default": { "type": "RDFString", "value": "" }, + "default": { + "type": "RDFString", + "value": "" + }, "doc": "Who started the flow.", "dynamic": false, "friendly_name": "Creator", @@ -6167,7 +9386,10 @@ "type": "RDFString" }, { - "default": { "type": "bool", "value": false }, + "default": { + "type": "bool", + "value": false + }, "doc": "Whether the flow was created by a GRRWorker.", "dynamic": false, "friendly_name": "Is robot", @@ -6208,13 +9430,18 @@ "dynamic": false, "friendly_name": "Original flow", "index": 13, - "labels": ["HIDDEN"], + "labels": [ + "HIDDEN" + ], "name": "original_flow", "repeated": false, "type": "ApiFlowReference" }, { - "default": { "type": "RDFString", "value": "" }, + "default": { + "type": "RDFString", + "value": "" + }, "doc": "An error that happened while reading the flow. This is not an error while executing the flow but an internal data issue.", "dynamic": false, "friendly_name": "Internal error", @@ -6225,7 +9452,13 @@ } ], "kind": "struct", - "mro": ["ApiFlow", "RDFProtoStruct", "RDFStruct", "RDFValue", "object"], + "mro": [ + "ApiFlow", + "RDFProtoStruct", + "RDFStruct", + "RDFValue", + "object" + ], "name": "ApiFlow" }, "test_class": "ApiGetRDFValueDescriptorHandlerRegressionTest_http_v1", @@ -6306,10 +9539,30 @@ }, { "allowed_values": [ - { "doc": "", "labels": [], "name": "RUNNING", "value": 0 }, - { "doc": "", "labels": [], "name": "TERMINATED", "value": 1 }, - { "doc": "", "labels": [], "name": "ERROR", "value": 3 }, - { "doc": "", "labels": [], "name": "CLIENT_CRASHED", "value": 4 } + { + "doc": "", + "labels": [], + "name": "RUNNING", + "value": 0 + }, + { + "doc": "", + "labels": [], + "name": "TERMINATED", + "value": 1 + }, + { + "doc": "", + "labels": [], + "name": "ERROR", + "value": 3 + }, + { + "doc": "", + "labels": [], + "name": "CLIENT_CRASHED", + "value": 4 + } ], "default": "RUNNING", "doc": "Current flow state.", @@ -6400,7 +9653,9 @@ "dynamic": false, "friendly_name": "Original flow", "index": 13, - "labels": ["HIDDEN"], + "labels": [ + "HIDDEN" + ], "name": "original_flow", "repeated": false, "type": "ApiFlowReference" @@ -6417,7 +9672,13 @@ } ], "kind": "struct", - "mro": ["ApiFlow", "RDFProtoStruct", "RDFStruct", "RDFValue", "object"], + "mro": [ + "ApiFlow", + "RDFProtoStruct", + "RDFStruct", + "RDFValue", + "object" + ], "name": "ApiFlow" }, "url": "/api/reflection/rdfvalue/ApiFlow" @@ -6442,34 +9703,61 @@ { "type": "ApiReportDataSeries2D", "value": { - "label": { "type": "unicode", "value": "Bar" }, + "label": { + "type": "unicode", + "value": "Bar" + }, "points": [ { "type": "ApiReportDataPoint2D", "value": { - "x": { "type": "long", "value": 5 }, - "y": { "type": "long", "value": 3 } + "x": { + "type": "long", + "value": 5 + }, + "y": { + "type": "long", + "value": 3 + } } }, { "type": "ApiReportDataPoint2D", "value": { - "x": { "type": "long", "value": 8 }, - "y": { "type": "long", "value": 4 } + "x": { + "type": "long", + "value": 8 + }, + "y": { + "type": "long", + "value": 4 + } } }, { "type": "ApiReportDataPoint2D", "value": { - "x": { "type": "long", "value": 13 }, - "y": { "type": "long", "value": 5 } + "x": { + "type": "long", + "value": 13 + }, + "y": { + "type": "long", + "value": 5 + } } }, { "type": "ApiReportDataPoint2D", "value": { - "x": { "type": "long", "value": 21 }, - "y": { "type": "long", "value": 6 } + "x": { + "type": "long", + "value": 21 + }, + "y": { + "type": "long", + "value": 6 + } } } ] @@ -6483,14 +9771,26 @@ "desc": { "type": "ApiReportDescriptor", "value": { - "name": { "type": "unicode", "value": "BarReportPlugin" }, - "requires_time_range": { "type": "bool", "value": true }, + "name": { + "type": "unicode", + "value": "BarReportPlugin" + }, + "requires_time_range": { + "type": "bool", + "value": true + }, "summary": { "type": "unicode", "value": "Reports bars' activity in the given time range." }, - "title": { "type": "unicode", "value": "Bar Activity" }, - "type": { "type": "EnumNamedValue", "value": "SERVER" } + "title": { + "type": "unicode", + "value": "Bar Activity" + }, + "type": { + "type": "EnumNamedValue", + "value": "SERVER" + } } } }, @@ -6503,10 +9803,22 @@ { "label": "Bar", "points": [ - { "x": 5, "y": 3 }, - { "x": 8, "y": 4 }, - { "x": 13, "y": 5 }, - { "x": 21, "y": 6 } + { + "x": 5, + "y": 3 + }, + { + "x": 8, + "y": 4 + }, + { + "x": 13, + "y": 5 + }, + { + "x": 21, + "y": 6 + } ] } ] @@ -6527,28 +9839,36 @@ { "api_method": "GetVfsFileContentUpdateState", "method": "GET", - "response": { "state": "RUNNING" }, + "response": { + "state": "RUNNING" + }, "test_class": "ApiGetVfsFileContentUpdateStateHandlerRegressionTest_http_v1", "url": "/api/clients/C.1000000000000000/vfs-update/ABCDEF" }, { "api_method": "GetVfsFileContentUpdateState", "method": "GET", - "response": { "state": "FINISHED" }, + "response": { + "state": "FINISHED" + }, "test_class": "ApiGetVfsFileContentUpdateStateHandlerRegressionTest_http_v1", "url": "/api/clients/C.1000000000000000/vfs-update/ABCDEF" }, { "api_method": "GetVfsFileContentUpdateState", "method": "GET", - "response": { "message": "Operation with id ABCDEF not found" }, + "response": { + "message": "Operation with id ABCDEF not found" + }, "test_class": "ApiGetVfsFileContentUpdateStateHandlerRegressionTest_http_v1", "url": "/api/clients/C.1000000000000000/vfs-update/ABCDEF" }, { "api_method": "GetVfsFileContentUpdateState", "method": "GET", - "response": { "message": "Flow id has incorrect format: `ABCDEF`" }, + "response": { + "message": "Flow id has incorrect format: `ABCDEF`" + }, "test_class": "ApiGetVfsFileContentUpdateStateHandlerRegressionTest_http_v1", "url": "/api/clients/C.1000000000000000/vfs-update/ABCDEF" } @@ -6557,28 +9877,36 @@ { "api_method": "GetVfsRefreshOperationState", "method": "GET", - "response": { "state": "RUNNING" }, + "response": { + "state": "RUNNING" + }, "test_class": "ApiGetVfsRefreshOperationStateHandlerRegressionTest_http_v1", "url": "/api/clients/C.1000000000000000/vfs-refresh-operations/ABCDEF" }, { "api_method": "GetVfsRefreshOperationState", "method": "GET", - "response": { "state": "FINISHED" }, + "response": { + "state": "FINISHED" + }, "test_class": "ApiGetVfsRefreshOperationStateHandlerRegressionTest_http_v1", "url": "/api/clients/C.1000000000000000/vfs-refresh-operations/ABCDEF" }, { "api_method": "GetVfsRefreshOperationState", "method": "GET", - "response": { "message": "Operation with id ABCDEF not found" }, + "response": { + "message": "Operation with id ABCDEF not found" + }, "test_class": "ApiGetVfsRefreshOperationStateHandlerRegressionTest_http_v1", "url": "/api/clients/C.1000000000000000/vfs-refresh-operations/ABCDEF" }, { "api_method": "GetVfsRefreshOperationState", "method": "GET", - "response": { "message": "Operation with id ABCDEF not found" }, + "response": { + "message": "Operation with id ABCDEF not found" + }, "test_class": "ApiGetVfsRefreshOperationStateHandlerRegressionTest_http_v1", "url": "/api/clients/C.1000000000000000/vfs-refresh-operations/ABCDEF" } @@ -6592,56 +9920,86 @@ { "type": "ApiVfsTimelineItem", "value": { - "action": { "type": "EnumNamedValue", "value": "MODIFICATION" }, + "action": { + "type": "EnumNamedValue", + "value": "MODIFICATION" + }, "file_path": { "type": "unicode", - "value": "fs/os/Users/\u4e2d\u56fd\u65b0\u95fb\u7f51\u65b0\u95fb\u4e2d/Shared/a.txt" + "value": "fs/os/Users/中国新闻网新闻中/Shared/a.txt" }, - "timestamp": { "type": "RDFDatetime", "value": 4000000 } + "timestamp": { + "type": "RDFDatetime", + "value": 4000000 + } } }, { "type": "ApiVfsTimelineItem", "value": { - "action": { "type": "EnumNamedValue", "value": "MODIFICATION" }, + "action": { + "type": "EnumNamedValue", + "value": "MODIFICATION" + }, "file_path": { "type": "unicode", - "value": "fs/os/Users/\u4e2d\u56fd\u65b0\u95fb\u7f51\u65b0\u95fb\u4e2d/Shared/a.txt" + "value": "fs/os/Users/中国新闻网新闻中/Shared/a.txt" }, - "timestamp": { "type": "RDFDatetime", "value": 3000000 } + "timestamp": { + "type": "RDFDatetime", + "value": 3000000 + } } }, { "type": "ApiVfsTimelineItem", "value": { - "action": { "type": "EnumNamedValue", "value": "MODIFICATION" }, + "action": { + "type": "EnumNamedValue", + "value": "MODIFICATION" + }, "file_path": { "type": "unicode", - "value": "fs/os/Users/\u4e2d\u56fd\u65b0\u95fb\u7f51\u65b0\u95fb\u4e2d/Shared/a.txt" + "value": "fs/os/Users/中国新闻网新闻中/Shared/a.txt" }, - "timestamp": { "type": "RDFDatetime", "value": 2000000 } + "timestamp": { + "type": "RDFDatetime", + "value": 2000000 + } } }, { "type": "ApiVfsTimelineItem", "value": { - "action": { "type": "EnumNamedValue", "value": "MODIFICATION" }, + "action": { + "type": "EnumNamedValue", + "value": "MODIFICATION" + }, "file_path": { "type": "unicode", - "value": "fs/os/Users/\u4e2d\u56fd\u65b0\u95fb\u7f51\u65b0\u95fb\u4e2d/Shared/a.txt" + "value": "fs/os/Users/中国新闻网新闻中/Shared/a.txt" }, - "timestamp": { "type": "RDFDatetime", "value": 1000000 } + "timestamp": { + "type": "RDFDatetime", + "value": 1000000 + } } }, { "type": "ApiVfsTimelineItem", "value": { - "action": { "type": "EnumNamedValue", "value": "MODIFICATION" }, + "action": { + "type": "EnumNamedValue", + "value": "MODIFICATION" + }, "file_path": { "type": "unicode", - "value": "fs/os/Users/\u4e2d\u56fd\u65b0\u95fb\u7f51\u65b0\u95fb\u4e2d/Shared/a.txt" + "value": "fs/os/Users/中国新闻网新闻中/Shared/a.txt" }, - "timestamp": { "type": "RDFDatetime", "value": 0 } + "timestamp": { + "type": "RDFDatetime", + "value": 0 + } } } ] @@ -6651,27 +10009,27 @@ "items": [ { "action": "MODIFICATION", - "file_path": "fs/os/Users/\u4e2d\u56fd\u65b0\u95fb\u7f51\u65b0\u95fb\u4e2d/Shared/a.txt", + "file_path": "fs/os/Users/中国新闻网新闻中/Shared/a.txt", "timestamp": 4000000 }, { "action": "MODIFICATION", - "file_path": "fs/os/Users/\u4e2d\u56fd\u65b0\u95fb\u7f51\u65b0\u95fb\u4e2d/Shared/a.txt", + "file_path": "fs/os/Users/中国新闻网新闻中/Shared/a.txt", "timestamp": 3000000 }, { "action": "MODIFICATION", - "file_path": "fs/os/Users/\u4e2d\u56fd\u65b0\u95fb\u7f51\u65b0\u95fb\u4e2d/Shared/a.txt", + "file_path": "fs/os/Users/中国新闻网新闻中/Shared/a.txt", "timestamp": 2000000 }, { "action": "MODIFICATION", - "file_path": "fs/os/Users/\u4e2d\u56fd\u65b0\u95fb\u7f51\u65b0\u95fb\u4e2d/Shared/a.txt", + "file_path": "fs/os/Users/中国新闻网新闻中/Shared/a.txt", "timestamp": 1000000 }, { "action": "MODIFICATION", - "file_path": "fs/os/Users/\u4e2d\u56fd\u65b0\u95fb\u7f51\u65b0\u95fb\u4e2d/Shared/a.txt", + "file_path": "fs/os/Users/中国新闻网新闻中/Shared/a.txt", "timestamp": 0 } ] @@ -6687,8 +10045,14 @@ "type": "ApiClientApproval", "value": { "approvers": [ - { "type": "unicode", "value": "api_test_user" }, - { "type": "unicode", "value": "requestor" } + { + "type": "unicode", + "value": "api_test_user" + }, + { + "type": "unicode", + "value": "requestor" + } ], "email_cc_addresses": [], "email_message_id": { @@ -6699,32 +10063,66 @@ "type": "RDFDatetime", "value": 2419244000000 }, - "id": { "type": "unicode", "value": "approval:111111" }, - "is_valid": { "type": "bool", "value": true }, - "notified_users": [{ "type": "unicode", "value": "api_test_user" }], - "reason": { "type": "unicode", "value": "foo" }, - "requestor": { "type": "unicode", "value": "requestor" }, - "subject": { - "type": "ApiClient", - "value": { - "age": { "type": "RDFDatetime", "value": 42000000 }, - "agent_info": { - "type": "ClientInformation", - "value": { - "build_time": { "type": "unicode", "value": "1980-01-01" }, - "client_name": { "type": "unicode", "value": "GRR Monitor" }, - "client_version": { "type": "long", "value": 1234 }, - "labels": [ - { "type": "unicode", "value": "label1" }, - { "type": "unicode", "value": "label2" } - ] - } - }, + "id": { + "type": "unicode", + "value": "approval:111111" + }, + "is_valid": { + "type": "bool", + "value": true + }, + "notified_users": [ + { + "type": "unicode", + "value": "api_test_user" + } + ], + "reason": { + "type": "unicode", + "value": "foo" + }, + "requestor": { + "type": "unicode", + "value": "requestor" + }, + "subject": { + "type": "ApiClient", + "value": { + "age": { + "type": "RDFDatetime", + "value": 42000000 + }, + "agent_info": { + "type": "ClientInformation", + "value": { + "build_time": { + "type": "unicode", + "value": "1980-01-01" + }, + "client_name": { + "type": "unicode", + "value": "GRR Monitor" + }, + "client_version": { + "type": "long", + "value": 1234 + }, + "labels": [ + { + "type": "unicode", + "value": "label1" + }, + { + "type": "unicode", + "value": "label2" + } + ] + } + }, "client_id": { "type": "ApiClientId", "value": "C.1000000000000000" }, - "fleetspeak_enabled": { "type": "bool", "value": false }, "hardware_info": { "type": "HardwareInfo", "value": { @@ -6770,55 +10168,100 @@ } } ], - "ifname": { "type": "unicode", "value": "if0" } + "ifname": { + "type": "unicode", + "value": "if0" + } } }, { "type": "Interface", "value": { - "ifname": { "type": "unicode", "value": "if1" }, - "mac_address": { "type": "MacAddress", "value": "qrvM3e4A" } + "ifname": { + "type": "unicode", + "value": "if1" + }, + "mac_address": { + "type": "MacAddress", + "value": "qrvM3e4A" + } } }, { "type": "Interface", "value": { - "ifname": { "type": "unicode", "value": "if2" }, - "mac_address": { "type": "MacAddress", "value": "u8zd7v8A" } + "ifname": { + "type": "unicode", + "value": "if2" + }, + "mac_address": { + "type": "MacAddress", + "value": "u8zd7v8A" + } } } ], "knowledge_base": { "type": "KnowledgeBase", "value": { - "fqdn": { "type": "unicode", "value": "Host-0.example.com" }, - "os": { "type": "unicode", "value": "Linux" }, + "fqdn": { + "type": "unicode", + "value": "Host-0.example.com" + }, + "os": { + "type": "unicode", + "value": "Linux" + }, "users": [ { "type": "User", "value": { - "username": { "type": "unicode", "value": "user1" } + "username": { + "type": "unicode", + "value": "user1" + } } }, { "type": "User", "value": { - "username": { "type": "unicode", "value": "user2" } + "username": { + "type": "unicode", + "value": "user2" + } } } ] } }, "labels": [], - "last_seen_at": { "type": "RDFDatetime", "value": 42000000 }, + "last_seen_at": { + "type": "RDFDatetime", + "value": 42000000 + }, "os_info": { "type": "Uname", "value": { - "fqdn": { "type": "unicode", "value": "Host-0.example.com" }, - "kernel": { "type": "unicode", "value": "4.0.0" }, - "machine": { "type": "unicode", "value": "x86_64" }, - "system": { "type": "unicode", "value": "Linux" }, - "version": { "type": "unicode", "value": "buster/sid" } + "fqdn": { + "type": "unicode", + "value": "Host-0.example.com" + }, + "kernel": { + "type": "unicode", + "value": "4.0.0" + }, + "machine": { + "type": "unicode", + "value": "x86_64" + }, + "system": { + "type": "unicode", + "value": "Linux" + }, + "version": { + "type": "unicode", + "value": "buster/sid" + } } }, "urn": { @@ -6829,13 +10272,19 @@ { "type": "User", "value": { - "username": { "type": "unicode", "value": "user1" } + "username": { + "type": "unicode", + "value": "user1" + } } }, { "type": "User", "value": { - "username": { "type": "unicode", "value": "user2" } + "username": { + "type": "unicode", + "value": "user2" + } } } ] @@ -6845,13 +10294,18 @@ }, "test_class": "ApiGrantClientApprovalHandlerRegressionTest_http_v1", "type_stripped_response": { - "approvers": ["api_test_user", "requestor"], + "approvers": [ + "api_test_user", + "requestor" + ], "email_cc_addresses": [], "email_message_id": "", "expiration_time_us": 2419244000000, "id": "approval:111111", "is_valid": true, - "notified_users": ["api_test_user"], + "notified_users": [ + "api_test_user" + ], "reason": "foo", "requestor": "requestor", "subject": { @@ -6860,10 +10314,12 @@ "build_time": "1980-01-01", "client_name": "GRR Monitor", "client_version": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "client_id": "C.1000000000000000", - "fleetspeak_enabled": false, "hardware_info": { "bios_version": "Bios-Version-0", "system_manufacturer": "System-Manufacturer-0" @@ -6871,7 +10327,10 @@ "interfaces": [ { "addresses": [ - { "address_type": "INET", "packed_bytes": "wKgAAA==" }, + { + "address_type": "INET", + "packed_bytes": "wKgAAA==" + }, { "address_type": "INET6", "packed_bytes": "IAGrzQAAAAAAAAAAAAAAAA==" @@ -6879,13 +10338,26 @@ ], "ifname": "if0" }, - { "ifname": "if1", "mac_address": "qrvM3e4A" }, - { "ifname": "if2", "mac_address": "u8zd7v8A" } + { + "ifname": "if1", + "mac_address": "qrvM3e4A" + }, + { + "ifname": "if2", + "mac_address": "u8zd7v8A" + } ], "knowledge_base": { "fqdn": "Host-0.example.com", "os": "Linux", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] }, "labels": [], "last_seen_at": 42000000, @@ -6897,7 +10369,14 @@ "version": "buster/sid" }, "urn": "aff4:/C.1000000000000000", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] } }, "url": "/api/users/requestor/approvals/client/C.1000000000000000/approval:111111/actions/grant" @@ -6911,23 +10390,49 @@ "type": "ApiCronJobApproval", "value": { "approvers": [ - { "type": "unicode", "value": "api_test_user" }, - { "type": "unicode", "value": "requestor" } + { + "type": "unicode", + "value": "api_test_user" + }, + { + "type": "unicode", + "value": "requestor" + } ], "email_cc_addresses": [], "email_message_id": { "type": "unicode", "value": "" }, - "id": { "type": "unicode", "value": "approval:111111" }, - "is_valid": { "type": "bool", "value": true }, - "notified_users": [{ "type": "unicode", "value": "api_test_user" }], - "reason": { "type": "unicode", "value": "foo" }, - "requestor": { "type": "unicode", "value": "requestor" }, + "id": { + "type": "unicode", + "value": "approval:111111" + }, + "is_valid": { + "type": "bool", + "value": true + }, + "notified_users": [ + { + "type": "unicode", + "value": "api_test_user" + } + ], + "reason": { + "type": "unicode", + "value": "foo" + }, + "requestor": { + "type": "unicode", + "value": "requestor" + }, "subject": { "type": "ApiCronJob", "value": { - "allow_overruns": { "type": "bool", "value": false }, + "allow_overruns": { + "type": "bool", + "value": false + }, "args": { "type": "CronJobAction", "value": { @@ -6938,7 +10443,10 @@ "hunt_cron_action": { "type": "HuntCronAction", "value": { - "flow_args": { "type": "InterrogateArgs", "value": {} }, + "flow_args": { + "type": "InterrogateArgs", + "value": {} + }, "flow_name": { "type": "unicode", "value": "Interrogate" @@ -6958,8 +10466,14 @@ "type": "long", "value": 1000 }, - "client_rate": { "type": "float", "value": 20.0 }, - "crash_limit": { "type": "long", "value": 100 } + "client_rate": { + "type": "float", + "value": 20.0 + }, + "crash_limit": { + "type": "long", + "value": 100 + } } } } @@ -6970,22 +10484,39 @@ "type": "ApiCronJobId", "value": "CronJob_123456" }, - "description": { "type": "unicode", "value": "" }, - "enabled": { "type": "bool", "value": true }, - "frequency": { "type": "DurationSeconds", "value": 86400 }, - "is_failing": { "type": "bool", "value": false } + "description": { + "type": "unicode", + "value": "" + }, + "enabled": { + "type": "bool", + "value": true + }, + "frequency": { + "type": "DurationSeconds", + "value": 86400 + }, + "is_failing": { + "type": "bool", + "value": false + } } } } }, "test_class": "ApiGrantCronJobApprovalHandlerRegressionTest_http_v1", "type_stripped_response": { - "approvers": ["api_test_user", "requestor"], + "approvers": [ + "api_test_user", + "requestor" + ], "email_cc_addresses": [], "email_message_id": "", "id": "approval:111111", "is_valid": true, - "notified_users": ["api_test_user"], + "notified_users": [ + "api_test_user" + ], "reason": "foo", "requestor": "requestor", "subject": { @@ -7022,8 +10553,14 @@ "type": "ApiHuntApproval", "value": { "approvers": [ - { "type": "unicode", "value": "api_test_user" }, - { "type": "unicode", "value": "requestor" } + { + "type": "unicode", + "value": "api_test_user" + }, + { + "type": "unicode", + "value": "requestor" + } ], "email_cc_addresses": [], "email_message_id": { @@ -7034,17 +10571,43 @@ "type": "RDFDatetime", "value": 2419244000000 }, - "id": { "type": "unicode", "value": "approval:111111" }, - "is_valid": { "type": "bool", "value": true }, - "notified_users": [{ "type": "unicode", "value": "api_test_user" }], - "reason": { "type": "unicode", "value": "foo" }, - "requestor": { "type": "unicode", "value": "requestor" }, + "id": { + "type": "unicode", + "value": "approval:111111" + }, + "is_valid": { + "type": "bool", + "value": true + }, + "notified_users": [ + { + "type": "unicode", + "value": "api_test_user" + } + ], + "reason": { + "type": "unicode", + "value": "foo" + }, + "requestor": { + "type": "unicode", + "value": "requestor" + }, "subject": { "type": "ApiHunt", "value": { - "all_clients_count": { "type": "long", "value": 0 }, - "client_limit": { "type": "long", "value": 100 }, - "client_rate": { "type": "float", "value": 0.0 }, + "all_clients_count": { + "type": "long", + "value": 0 + }, + "client_limit": { + "type": "long", + "value": 100 + }, + "client_rate": { + "type": "float", + "value": 0.0 + }, "client_rule_set": { "type": "ForemanClientRuleSet", "value": { @@ -7074,29 +10637,68 @@ ] } }, - "clients_with_results_count": { "type": "long", "value": 0 }, - "completed_clients_count": { "type": "long", "value": 0 }, - "crash_limit": { "type": "long", "value": 100 }, - "crashed_clients_count": { "type": "long", "value": 0 }, - "created": { "type": "RDFDatetime", "value": 42000000 }, - "creator": { "type": "unicode", "value": "api_test_user" }, - "description": { "type": "unicode", "value": "a hunt" }, - "duration": { "type": "DurationSeconds", "value": 1209600 }, - "failed_clients_count": { "type": "long", "value": 0 }, + "clients_with_results_count": { + "type": "long", + "value": 0 + }, + "completed_clients_count": { + "type": "long", + "value": 0 + }, + "crash_limit": { + "type": "long", + "value": 100 + }, + "crashed_clients_count": { + "type": "long", + "value": 0 + }, + "created": { + "type": "RDFDatetime", + "value": 42000000 + }, + "creator": { + "type": "unicode", + "value": "api_test_user" + }, + "description": { + "type": "unicode", + "value": "a hunt" + }, + "duration": { + "type": "DurationSeconds", + "value": 1209600 + }, + "failed_clients_count": { + "type": "long", + "value": 0 + }, "flow_args": { "type": "GetFileArgs", "value": { "pathspec": { "type": "PathSpec", "value": { - "path": { "type": "unicode", "value": "/tmp/evil.txt" }, - "pathtype": { "type": "EnumNamedValue", "value": "OS" } + "path": { + "type": "unicode", + "value": "/tmp/evil.txt" + }, + "pathtype": { + "type": "EnumNamedValue", + "value": "OS" + } } } } }, - "flow_name": { "type": "unicode", "value": "GetFile" }, - "hunt_id": { "type": "ApiHuntId", "value": "H:123456" }, + "flow_name": { + "type": "unicode", + "value": "GetFile" + }, + "hunt_id": { + "type": "ApiHuntId", + "value": "H:123456" + }, "hunt_runner_args": { "type": "HuntRunnerArgs", "value": { @@ -7112,7 +10714,10 @@ "type": "long", "value": 1000 }, - "client_rate": { "type": "float", "value": 0.0 }, + "client_rate": { + "type": "float", + "value": 0.0 + }, "client_rule_set": { "type": "ForemanClientRuleSet", "value": { @@ -7142,42 +10747,90 @@ ] } }, - "crash_limit": { "type": "long", "value": 100 }, - "description": { "type": "unicode", "value": "a hunt" }, + "crash_limit": { + "type": "long", + "value": 100 + }, + "description": { + "type": "unicode", + "value": "a hunt" + }, "expiry_time": { "type": "DurationSeconds", "value": 1209600 }, - "hunt_name": { "type": "unicode", "value": "GenericHunt" }, + "hunt_name": { + "type": "unicode", + "value": "GenericHunt" + }, "original_object": { "type": "FlowLikeObjectReference", "value": {} } } }, - "hunt_type": { "type": "EnumNamedValue", "value": "STANDARD" }, - "is_robot": { "type": "bool", "value": false }, - "name": { "type": "unicode", "value": "GenericHunt" }, - "remaining_clients_count": { "type": "long", "value": 0 }, - "results_count": { "type": "long", "value": 0 }, - "state": { "type": "EnumNamedValue", "value": "PAUSED" }, - "state_comment": { "type": "unicode", "value": "" }, - "total_cpu_usage": { "type": "long", "value": 0 }, - "total_net_usage": { "type": "long", "value": 0 }, - "urn": { "type": "SessionID", "value": "aff4:/hunts/H:123456" } + "hunt_type": { + "type": "EnumNamedValue", + "value": "STANDARD" + }, + "is_robot": { + "type": "bool", + "value": false + }, + "name": { + "type": "unicode", + "value": "GenericHunt" + }, + "remaining_clients_count": { + "type": "long", + "value": 0 + }, + "results_count": { + "type": "long", + "value": 0 + }, + "state": { + "type": "EnumNamedValue", + "value": "PAUSED" + }, + "state_comment": { + "type": "unicode", + "value": "" + }, + "state_reason": { + "type": "EnumNamedValue", + "value": "UNKNOWN" + }, + "total_cpu_usage": { + "type": "long", + "value": 0 + }, + "total_net_usage": { + "type": "long", + "value": 0 + }, + "urn": { + "type": "SessionID", + "value": "aff4:/hunts/H:123456" + } } } } }, "test_class": "ApiGrantHuntApprovalHandlerRegressionTest_http_v1", "type_stripped_response": { - "approvers": ["api_test_user", "requestor"], + "approvers": [ + "api_test_user", + "requestor" + ], "email_cc_addresses": [], "email_message_id": "", "expiration_time_us": 2419244000000, "id": "approval:111111", "is_valid": true, - "notified_users": ["api_test_user"], + "notified_users": [ + "api_test_user" + ], "reason": "foo", "requestor": "requestor", "subject": { @@ -7187,7 +10840,10 @@ "client_rule_set": { "rules": [ { - "regex": { "attribute_regex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attribute_regex": "GRR", + "field": "CLIENT_NAME" + }, "rule_type": "REGEX" } ] @@ -7202,7 +10858,10 @@ "duration": 1209600, "failed_clients_count": 0, "flow_args": { - "pathspec": { "path": "/tmp/evil.txt", "pathtype": "OS" } + "pathspec": { + "path": "/tmp/evil.txt", + "pathtype": "OS" + } }, "flow_name": "GetFile", "hunt_id": "H:123456", @@ -7214,7 +10873,10 @@ "client_rule_set": { "rules": [ { - "regex": { "attribute_regex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attribute_regex": "GRR", + "field": "CLIENT_NAME" + }, "rule_type": "REGEX" } ] @@ -7232,6 +10894,7 @@ "results_count": 0, "state": "PAUSED", "state_comment": "", + "state_reason": "UNKNOWN", "total_cpu_usage": 0, "total_net_usage": 0, "urn": "aff4:/hunts/H:123456" @@ -7249,7 +10912,10 @@ { "type": "ApiNotification", "value": { - "is_pending": { "type": "bool", "value": true }, + "is_pending": { + "type": "bool", + "value": true + }, "message": { "type": "unicode", "value": "Host-0.example.com: " @@ -7261,7 +10927,10 @@ "reference": { "type": "ApiNotificationReference", "value": { - "type": { "type": "EnumNamedValue", "value": "VFS" }, + "type": { + "type": "EnumNamedValue", + "value": "VFS" + }, "vfs": { "type": "ApiNotificationVfsReference", "value": { @@ -7269,18 +10938,27 @@ "type": "ApiClientId", "value": "C.1000000000000000" }, - "vfs_path": { "type": "unicode", "value": "fs/os/foo" } + "vfs_path": { + "type": "unicode", + "value": "fs/os/foo" + } } } } }, - "timestamp": { "type": "RDFDatetime", "value": 44000000 } + "timestamp": { + "type": "RDFDatetime", + "value": 44000000 + } } }, { "type": "ApiNotification", "value": { - "is_pending": { "type": "bool", "value": true }, + "is_pending": { + "type": "bool", + "value": true + }, "message": { "type": "unicode", "value": "Host-0.example.com: " @@ -7301,10 +10979,16 @@ } } }, - "type": { "type": "EnumNamedValue", "value": "CLIENT" } + "type": { + "type": "EnumNamedValue", + "value": "CLIENT" + } } }, - "timestamp": { "type": "RDFDatetime", "value": 42000000 } + "timestamp": { + "type": "RDFDatetime", + "value": 42000000 + } } } ], @@ -7331,7 +11015,9 @@ "message": "Host-0.example.com: ", "notification_type": "TYPE_CLIENT_INTERROGATED", "reference": { - "client": { "client_id": "C.1000000000000000" }, + "client": { + "client_id": "C.1000000000000000" + }, "type": "CLIENT" }, "timestamp": 42000000 @@ -7344,13 +11030,19 @@ { "api_method": "ListAndResetUserNotifications", "method": "POST", - "request_payload": { "count": 1, "offset": 1 }, + "request_payload": { + "count": 1, + "offset": 1 + }, "response": { "items": [ { "type": "ApiNotification", "value": { - "is_pending": { "type": "bool", "value": false }, + "is_pending": { + "type": "bool", + "value": false + }, "message": { "type": "unicode", "value": "Host-0.example.com: " @@ -7371,10 +11063,16 @@ } } }, - "type": { "type": "EnumNamedValue", "value": "CLIENT" } + "type": { + "type": "EnumNamedValue", + "value": "CLIENT" + } } }, - "timestamp": { "type": "RDFDatetime", "value": 42000000 } + "timestamp": { + "type": "RDFDatetime", + "value": 42000000 + } } } ], @@ -7388,7 +11086,9 @@ "message": "Host-0.example.com: ", "notification_type": "TYPE_CLIENT_INTERROGATED", "reference": { - "client": { "client_id": "C.1000000000000000" }, + "client": { + "client_id": "C.1000000000000000" + }, "type": "CLIENT" }, "timestamp": 42000000 @@ -7401,13 +11101,18 @@ { "api_method": "ListAndResetUserNotifications", "method": "POST", - "request_payload": { "filter": "other" }, + "request_payload": { + "filter": "other" + }, "response": { "items": [ { "type": "ApiNotification", "value": { - "is_pending": { "type": "bool", "value": false }, + "is_pending": { + "type": "bool", + "value": false + }, "message": { "type": "unicode", "value": "Host-0.example.com: " @@ -7419,7 +11124,10 @@ "reference": { "type": "ApiNotificationReference", "value": { - "type": { "type": "EnumNamedValue", "value": "VFS" }, + "type": { + "type": "EnumNamedValue", + "value": "VFS" + }, "vfs": { "type": "ApiNotificationVfsReference", "value": { @@ -7427,12 +11135,18 @@ "type": "ApiClientId", "value": "C.1000000000000000" }, - "vfs_path": { "type": "unicode", "value": "fs/os/foo" } + "vfs_path": { + "type": "unicode", + "value": "fs/os/foo" + } } } } }, - "timestamp": { "type": "RDFDatetime", "value": 44000000 } + "timestamp": { + "type": "RDFDatetime", + "value": 44000000 + } } } ], @@ -7464,7 +11178,9 @@ { "api_method": "ListApproverSuggestions", "method": "GET", - "response": { "suggestions": [] }, + "response": { + "suggestions": [] + }, "test_class": "ApiListApproverSuggestionsHandlerRegressionTest_http_v1", "url": "/api/users/approver-suggestions?username_query=foo" }, @@ -7476,19 +11192,28 @@ { "type": "ApproverSuggestion", "value": { - "username": { "type": "unicode", "value": "sanchezmorty" } + "username": { + "type": "unicode", + "value": "sanchezmorty" + } } }, { "type": "ApproverSuggestion", "value": { - "username": { "type": "unicode", "value": "sanchezrick" } + "username": { + "type": "unicode", + "value": "sanchezrick" + } } }, { "type": "ApproverSuggestion", "value": { - "username": { "type": "unicode", "value": "sanchezsummer" } + "username": { + "type": "unicode", + "value": "sanchezsummer" + } } } ] @@ -7496,9 +11221,15 @@ "test_class": "ApiListApproverSuggestionsHandlerRegressionTest_http_v1", "type_stripped_response": { "suggestions": [ - { "username": "sanchezmorty" }, - { "username": "sanchezrick" }, - { "username": "sanchezsummer" } + { + "username": "sanchezmorty" + }, + { + "username": "sanchezrick" + }, + { + "username": "sanchezsummer" + } ] }, "url": "/api/users/approver-suggestions?username_query=sanchez" @@ -7516,14 +11247,14 @@ "artifact": { "type": "Artifact", "value": { - "conditions": [ - { "type": "unicode", "value": "os_major_version >= 6" } - ], "doc": { "type": "unicode", "value": "Extract the installed drivers on Windows via WMI." }, - "name": { "type": "ArtifactName", "value": "TestDrivers" }, + "name": { + "type": "ArtifactName", + "value": "TestDrivers" + }, "provides": [], "sources": [ { @@ -7538,21 +11269,37 @@ } } }, - "conditions": [], "returned_types": [], - "type": { "type": "EnumNamedValue", "value": "WMI" } + "type": { + "type": "EnumNamedValue", + "value": "WMI" + } } } ], - "supported_os": [{ "type": "unicode", "value": "Windows" }], + "supported_os": [ + { + "type": "unicode", + "value": "Windows" + } + ], "urls": [ - { "type": "unicode", "value": "http://www.example.com" } + { + "type": "unicode", + "value": "http://www.example.com" + } ] } }, "dependencies": [], - "error_message": { "type": "unicode", "value": "" }, - "is_custom": { "type": "bool", "value": false }, + "error_message": { + "type": "unicode", + "value": "" + }, + "is_custom": { + "type": "bool", + "value": false + }, "path_dependencies": [] } } @@ -7564,20 +11311,24 @@ "items": [ { "artifact": { - "conditions": ["os_major_version >= 6"], "doc": "Extract the installed drivers on Windows via WMI.", "name": "TestDrivers", "provides": [], "sources": [ { - "attributes": { "query": "SELECT * from Win32_SystemDriver" }, - "conditions": [], + "attributes": { + "query": "SELECT * from Win32_SystemDriver" + }, "returned_types": [], "type": "WMI" } ], - "supported_os": ["Windows"], - "urls": ["http://www.example.com"] + "supported_os": [ + "Windows" + ], + "urls": [ + "http://www.example.com" + ] }, "dependencies": [], "error_message": "", @@ -7594,123 +11345,15 @@ { "api_method": "ListClientActionRequests", "method": "GET", - "response": { - "items": [ - { - "type": "ApiClientActionRequest", - "value": { - "client_action": { "type": "unicode", "value": "ListProcesses" }, - "leased_until": { "type": "RDFDatetime", "value": 10042000000 }, - "session_id": { - "type": "RDFURN", - "value": "aff4:/C.1000000000000000/flows/W:ABCDEF" - } - } - } - ] - }, + "response": {}, "test_class": "ApiListClientActionRequestsHandlerRegressionTest_http_v1", - "type_stripped_response": { - "items": [ - { - "client_action": "ListProcesses", - "leased_until": 10042000000, - "session_id": "aff4:/C.1000000000000000/flows/W:ABCDEF" - } - ] - }, "url": "/api/clients/C.1000000000000000/action-requests" }, { "api_method": "ListClientActionRequests", "method": "GET", - "response": { - "items": [ - { - "type": "ApiClientActionRequest", - "value": { - "client_action": { "type": "unicode", "value": "ListProcesses" }, - "leased_until": { "type": "RDFDatetime", "value": 10042000000 }, - "responses": [ - { - "type": "GrrMessage", - "value": { - "payload": { - "type": "Process", - "value": { - "name": { "type": "unicode", "value": "test_process" } - } - }, - "payload_type": { "type": "unicode", "value": "Process" }, - "request_id": { "type": "long", "value": 1 }, - "response_id": { "type": "long", "value": 1 }, - "session_id": { - "type": "FlowSessionID", - "value": "aff4:/C.1000000000000000/flows/W:ABCDEF" - }, - "timestamp": { "type": "RDFDatetime", "value": 42000000 }, - "type": { "type": "EnumNamedValue", "value": "MESSAGE" } - } - }, - { - "type": "GrrMessage", - "value": { - "payload": { - "type": "GrrStatus", - "value": { - "status": { "type": "EnumNamedValue", "value": "OK" } - } - }, - "payload_type": { "type": "unicode", "value": "GrrStatus" }, - "request_id": { "type": "long", "value": 1 }, - "response_id": { "type": "long", "value": 2 }, - "session_id": { - "type": "FlowSessionID", - "value": "aff4:/C.1000000000000000/flows/W:ABCDEF" - }, - "timestamp": { "type": "RDFDatetime", "value": 42000000 }, - "type": { "type": "EnumNamedValue", "value": "STATUS" } - } - } - ], - "session_id": { - "type": "RDFURN", - "value": "aff4:/C.1000000000000000/flows/W:ABCDEF" - } - } - } - ] - }, + "response": {}, "test_class": "ApiListClientActionRequestsHandlerRegressionTest_http_v1", - "type_stripped_response": { - "items": [ - { - "client_action": "ListProcesses", - "leased_until": 10042000000, - "responses": [ - { - "payload": { "name": "test_process" }, - "payload_type": "Process", - "request_id": 1, - "response_id": 1, - "session_id": "aff4:/C.1000000000000000/flows/W:ABCDEF", - "timestamp": 42000000, - "type": "MESSAGE" - }, - { - "payload": { "status": "OK" }, - "payload_type": "GrrStatus", - "request_id": 1, - "response_id": 2, - "session_id": "aff4:/C.1000000000000000/flows/W:ABCDEF", - "timestamp": 42000000, - "type": "STATUS" - } - ], - "session_id": "aff4:/C.1000000000000000/flows/W:ABCDEF" - } - ] - }, "url": "/api/clients/C.1000000000000000/action-requests?fetch_responses=1" } ], @@ -7724,8 +11367,14 @@ "type": "ApiClientApproval", "value": { "approvers": [ - { "type": "unicode", "value": "api_test_user" }, - { "type": "unicode", "value": "approver" } + { + "type": "unicode", + "value": "api_test_user" + }, + { + "type": "unicode", + "value": "approver" + } ], "email_cc_addresses": [], "email_message_id": { @@ -7736,15 +11385,35 @@ "type": "RDFDatetime", "value": 2419245000000 }, - "id": { "type": "unicode", "value": "approval:222222" }, - "is_valid": { "type": "bool", "value": true }, - "notified_users": [{ "type": "unicode", "value": "approver" }], - "reason": { "type": "unicode", "value": "Running tests" }, - "requestor": { "type": "unicode", "value": "api_test_user" }, + "id": { + "type": "unicode", + "value": "approval:222222" + }, + "is_valid": { + "type": "bool", + "value": true + }, + "notified_users": [ + { + "type": "unicode", + "value": "approver" + } + ], + "reason": { + "type": "unicode", + "value": "Running tests" + }, + "requestor": { + "type": "unicode", + "value": "api_test_user" + }, "subject": { "type": "ApiClient", "value": { - "age": { "type": "RDFDatetime", "value": 42000000 }, + "age": { + "type": "RDFDatetime", + "value": 42000000 + }, "agent_info": { "type": "ClientInformation", "value": { @@ -7756,10 +11425,19 @@ "type": "unicode", "value": "GRR Monitor" }, - "client_version": { "type": "long", "value": 1234 }, + "client_version": { + "type": "long", + "value": 1234 + }, "labels": [ - { "type": "unicode", "value": "label1" }, - { "type": "unicode", "value": "label2" } + { + "type": "unicode", + "value": "label1" + }, + { + "type": "unicode", + "value": "label2" + } ] } }, @@ -7767,7 +11445,6 @@ "type": "ApiClientId", "value": "C.1000000000000001" }, - "fleetspeak_enabled": { "type": "bool", "value": false }, "hardware_info": { "type": "HardwareInfo", "value": { @@ -7813,13 +11490,19 @@ } } ], - "ifname": { "type": "unicode", "value": "if0" } + "ifname": { + "type": "unicode", + "value": "if0" + } } }, { "type": "Interface", "value": { - "ifname": { "type": "unicode", "value": "if1" }, + "ifname": { + "type": "unicode", + "value": "if1" + }, "mac_address": { "type": "MacAddress", "value": "qrvM3e4B" @@ -7829,7 +11512,10 @@ { "type": "Interface", "value": { - "ifname": { "type": "unicode", "value": "if2" }, + "ifname": { + "type": "unicode", + "value": "if2" + }, "mac_address": { "type": "MacAddress", "value": "u8zd7v8B" @@ -7844,25 +11530,37 @@ "type": "unicode", "value": "Host-1.example.com" }, - "os": { "type": "unicode", "value": "Linux" }, + "os": { + "type": "unicode", + "value": "Linux" + }, "users": [ { "type": "User", "value": { - "username": { "type": "unicode", "value": "user1" } + "username": { + "type": "unicode", + "value": "user1" + } } }, { "type": "User", "value": { - "username": { "type": "unicode", "value": "user2" } + "username": { + "type": "unicode", + "value": "user2" + } } } ] } }, "labels": [], - "last_seen_at": { "type": "RDFDatetime", "value": 42000000 }, + "last_seen_at": { + "type": "RDFDatetime", + "value": 42000000 + }, "os_info": { "type": "Uname", "value": { @@ -7870,10 +11568,22 @@ "type": "unicode", "value": "Host-1.example.com" }, - "kernel": { "type": "unicode", "value": "4.0.0" }, - "machine": { "type": "unicode", "value": "x86_64" }, - "system": { "type": "unicode", "value": "Linux" }, - "version": { "type": "unicode", "value": "buster/sid" } + "kernel": { + "type": "unicode", + "value": "4.0.0" + }, + "machine": { + "type": "unicode", + "value": "x86_64" + }, + "system": { + "type": "unicode", + "value": "Linux" + }, + "version": { + "type": "unicode", + "value": "buster/sid" + } } }, "urn": { @@ -7884,13 +11594,19 @@ { "type": "User", "value": { - "username": { "type": "unicode", "value": "user1" } + "username": { + "type": "unicode", + "value": "user1" + } } }, { "type": "User", "value": { - "username": { "type": "unicode", "value": "user2" } + "username": { + "type": "unicode", + "value": "user2" + } } } ] @@ -7901,7 +11617,12 @@ { "type": "ApiClientApproval", "value": { - "approvers": [{ "type": "unicode", "value": "api_test_user" }], + "approvers": [ + { + "type": "unicode", + "value": "api_test_user" + } + ], "email_cc_addresses": [], "email_message_id": { "type": "unicode", @@ -7911,19 +11632,39 @@ "type": "RDFDatetime", "value": 2419244000000 }, - "id": { "type": "unicode", "value": "approval:111111" }, - "is_valid": { "type": "bool", "value": false }, + "id": { + "type": "unicode", + "value": "approval:111111" + }, + "is_valid": { + "type": "bool", + "value": false + }, "is_valid_message": { "type": "unicode", "value": "Need at least 1 additional approver for access." }, - "notified_users": [{ "type": "unicode", "value": "approver" }], - "reason": { "type": "unicode", "value": "Running tests" }, - "requestor": { "type": "unicode", "value": "api_test_user" }, + "notified_users": [ + { + "type": "unicode", + "value": "approver" + } + ], + "reason": { + "type": "unicode", + "value": "Running tests" + }, + "requestor": { + "type": "unicode", + "value": "api_test_user" + }, "subject": { "type": "ApiClient", "value": { - "age": { "type": "RDFDatetime", "value": 42000000 }, + "age": { + "type": "RDFDatetime", + "value": 42000000 + }, "agent_info": { "type": "ClientInformation", "value": { @@ -7935,10 +11676,19 @@ "type": "unicode", "value": "GRR Monitor" }, - "client_version": { "type": "long", "value": 1234 }, + "client_version": { + "type": "long", + "value": 1234 + }, "labels": [ - { "type": "unicode", "value": "label1" }, - { "type": "unicode", "value": "label2" } + { + "type": "unicode", + "value": "label1" + }, + { + "type": "unicode", + "value": "label2" + } ] } }, @@ -7946,7 +11696,6 @@ "type": "ApiClientId", "value": "C.1000000000000000" }, - "fleetspeak_enabled": { "type": "bool", "value": false }, "hardware_info": { "type": "HardwareInfo", "value": { @@ -7992,13 +11741,19 @@ } } ], - "ifname": { "type": "unicode", "value": "if0" } + "ifname": { + "type": "unicode", + "value": "if0" + } } }, { "type": "Interface", "value": { - "ifname": { "type": "unicode", "value": "if1" }, + "ifname": { + "type": "unicode", + "value": "if1" + }, "mac_address": { "type": "MacAddress", "value": "qrvM3e4A" @@ -8008,7 +11763,10 @@ { "type": "Interface", "value": { - "ifname": { "type": "unicode", "value": "if2" }, + "ifname": { + "type": "unicode", + "value": "if2" + }, "mac_address": { "type": "MacAddress", "value": "u8zd7v8A" @@ -8023,25 +11781,37 @@ "type": "unicode", "value": "Host-0.example.com" }, - "os": { "type": "unicode", "value": "Linux" }, + "os": { + "type": "unicode", + "value": "Linux" + }, "users": [ { "type": "User", "value": { - "username": { "type": "unicode", "value": "user1" } + "username": { + "type": "unicode", + "value": "user1" + } } }, { "type": "User", "value": { - "username": { "type": "unicode", "value": "user2" } + "username": { + "type": "unicode", + "value": "user2" + } } } ] } }, "labels": [], - "last_seen_at": { "type": "RDFDatetime", "value": 42000000 }, + "last_seen_at": { + "type": "RDFDatetime", + "value": 42000000 + }, "os_info": { "type": "Uname", "value": { @@ -8049,10 +11819,22 @@ "type": "unicode", "value": "Host-0.example.com" }, - "kernel": { "type": "unicode", "value": "4.0.0" }, - "machine": { "type": "unicode", "value": "x86_64" }, - "system": { "type": "unicode", "value": "Linux" }, - "version": { "type": "unicode", "value": "buster/sid" } + "kernel": { + "type": "unicode", + "value": "4.0.0" + }, + "machine": { + "type": "unicode", + "value": "x86_64" + }, + "system": { + "type": "unicode", + "value": "Linux" + }, + "version": { + "type": "unicode", + "value": "buster/sid" + } } }, "urn": { @@ -8063,13 +11845,19 @@ { "type": "User", "value": { - "username": { "type": "unicode", "value": "user1" } + "username": { + "type": "unicode", + "value": "user1" + } } }, { "type": "User", "value": { - "username": { "type": "unicode", "value": "user2" } + "username": { + "type": "unicode", + "value": "user2" + } } } ] @@ -8083,13 +11871,18 @@ "type_stripped_response": { "items": [ { - "approvers": ["api_test_user", "approver"], + "approvers": [ + "api_test_user", + "approver" + ], "email_cc_addresses": [], "email_message_id": "", "expiration_time_us": 2419245000000, "id": "approval:222222", "is_valid": true, - "notified_users": ["approver"], + "notified_users": [ + "approver" + ], "reason": "Running tests", "requestor": "api_test_user", "subject": { @@ -8098,10 +11891,12 @@ "build_time": "1980-01-01", "client_name": "GRR Monitor", "client_version": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "client_id": "C.1000000000000001", - "fleetspeak_enabled": false, "hardware_info": { "bios_version": "Bios-Version-1", "system_manufacturer": "System-Manufacturer-1" @@ -8109,7 +11904,10 @@ "interfaces": [ { "addresses": [ - { "address_type": "INET", "packed_bytes": "wKgAAQ==" }, + { + "address_type": "INET", + "packed_bytes": "wKgAAQ==" + }, { "address_type": "INET6", "packed_bytes": "IAGrzQAAAAAAAAAAAAAAAQ==" @@ -8117,13 +11915,26 @@ ], "ifname": "if0" }, - { "ifname": "if1", "mac_address": "qrvM3e4B" }, - { "ifname": "if2", "mac_address": "u8zd7v8B" } + { + "ifname": "if1", + "mac_address": "qrvM3e4B" + }, + { + "ifname": "if2", + "mac_address": "u8zd7v8B" + } ], "knowledge_base": { "fqdn": "Host-1.example.com", "os": "Linux", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] }, "labels": [], "last_seen_at": 42000000, @@ -8135,18 +11946,29 @@ "version": "buster/sid" }, "urn": "aff4:/C.1000000000000001", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] } }, { - "approvers": ["api_test_user"], + "approvers": [ + "api_test_user" + ], "email_cc_addresses": [], "email_message_id": "", "expiration_time_us": 2419244000000, "id": "approval:111111", "is_valid": false, "is_valid_message": "Need at least 1 additional approver for access.", - "notified_users": ["approver"], + "notified_users": [ + "approver" + ], "reason": "Running tests", "requestor": "api_test_user", "subject": { @@ -8155,10 +11977,12 @@ "build_time": "1980-01-01", "client_name": "GRR Monitor", "client_version": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "client_id": "C.1000000000000000", - "fleetspeak_enabled": false, "hardware_info": { "bios_version": "Bios-Version-0", "system_manufacturer": "System-Manufacturer-0" @@ -8166,7 +11990,10 @@ "interfaces": [ { "addresses": [ - { "address_type": "INET", "packed_bytes": "wKgAAA==" }, + { + "address_type": "INET", + "packed_bytes": "wKgAAA==" + }, { "address_type": "INET6", "packed_bytes": "IAGrzQAAAAAAAAAAAAAAAA==" @@ -8174,13 +12001,26 @@ ], "ifname": "if0" }, - { "ifname": "if1", "mac_address": "qrvM3e4A" }, - { "ifname": "if2", "mac_address": "u8zd7v8A" } + { + "ifname": "if1", + "mac_address": "qrvM3e4A" + }, + { + "ifname": "if2", + "mac_address": "u8zd7v8A" + } ], "knowledge_base": { "fqdn": "Host-0.example.com", "os": "Linux", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] }, "labels": [], "last_seen_at": 42000000, @@ -8192,7 +12032,14 @@ "version": "buster/sid" }, "urn": "aff4:/C.1000000000000000", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] } } ] @@ -8207,7 +12054,12 @@ { "type": "ApiClientApproval", "value": { - "approvers": [{ "type": "unicode", "value": "api_test_user" }], + "approvers": [ + { + "type": "unicode", + "value": "api_test_user" + } + ], "email_cc_addresses": [], "email_message_id": { "type": "unicode", @@ -8217,19 +12069,39 @@ "type": "RDFDatetime", "value": 2419244000000 }, - "id": { "type": "unicode", "value": "approval:111111" }, - "is_valid": { "type": "bool", "value": false }, + "id": { + "type": "unicode", + "value": "approval:111111" + }, + "is_valid": { + "type": "bool", + "value": false + }, "is_valid_message": { "type": "unicode", "value": "Need at least 1 additional approver for access." }, - "notified_users": [{ "type": "unicode", "value": "approver" }], - "reason": { "type": "unicode", "value": "Running tests" }, - "requestor": { "type": "unicode", "value": "api_test_user" }, + "notified_users": [ + { + "type": "unicode", + "value": "approver" + } + ], + "reason": { + "type": "unicode", + "value": "Running tests" + }, + "requestor": { + "type": "unicode", + "value": "api_test_user" + }, "subject": { "type": "ApiClient", "value": { - "age": { "type": "RDFDatetime", "value": 42000000 }, + "age": { + "type": "RDFDatetime", + "value": 42000000 + }, "agent_info": { "type": "ClientInformation", "value": { @@ -8241,10 +12113,19 @@ "type": "unicode", "value": "GRR Monitor" }, - "client_version": { "type": "long", "value": 1234 }, + "client_version": { + "type": "long", + "value": 1234 + }, "labels": [ - { "type": "unicode", "value": "label1" }, - { "type": "unicode", "value": "label2" } + { + "type": "unicode", + "value": "label1" + }, + { + "type": "unicode", + "value": "label2" + } ] } }, @@ -8252,7 +12133,6 @@ "type": "ApiClientId", "value": "C.1000000000000000" }, - "fleetspeak_enabled": { "type": "bool", "value": false }, "hardware_info": { "type": "HardwareInfo", "value": { @@ -8298,13 +12178,19 @@ } } ], - "ifname": { "type": "unicode", "value": "if0" } + "ifname": { + "type": "unicode", + "value": "if0" + } } }, { "type": "Interface", "value": { - "ifname": { "type": "unicode", "value": "if1" }, + "ifname": { + "type": "unicode", + "value": "if1" + }, "mac_address": { "type": "MacAddress", "value": "qrvM3e4A" @@ -8314,7 +12200,10 @@ { "type": "Interface", "value": { - "ifname": { "type": "unicode", "value": "if2" }, + "ifname": { + "type": "unicode", + "value": "if2" + }, "mac_address": { "type": "MacAddress", "value": "u8zd7v8A" @@ -8329,25 +12218,37 @@ "type": "unicode", "value": "Host-0.example.com" }, - "os": { "type": "unicode", "value": "Linux" }, + "os": { + "type": "unicode", + "value": "Linux" + }, "users": [ { "type": "User", "value": { - "username": { "type": "unicode", "value": "user1" } + "username": { + "type": "unicode", + "value": "user1" + } } }, { "type": "User", "value": { - "username": { "type": "unicode", "value": "user2" } + "username": { + "type": "unicode", + "value": "user2" + } } } ] } }, "labels": [], - "last_seen_at": { "type": "RDFDatetime", "value": 42000000 }, + "last_seen_at": { + "type": "RDFDatetime", + "value": 42000000 + }, "os_info": { "type": "Uname", "value": { @@ -8355,10 +12256,22 @@ "type": "unicode", "value": "Host-0.example.com" }, - "kernel": { "type": "unicode", "value": "4.0.0" }, - "machine": { "type": "unicode", "value": "x86_64" }, - "system": { "type": "unicode", "value": "Linux" }, - "version": { "type": "unicode", "value": "buster/sid" } + "kernel": { + "type": "unicode", + "value": "4.0.0" + }, + "machine": { + "type": "unicode", + "value": "x86_64" + }, + "system": { + "type": "unicode", + "value": "Linux" + }, + "version": { + "type": "unicode", + "value": "buster/sid" + } } }, "urn": { @@ -8369,13 +12282,19 @@ { "type": "User", "value": { - "username": { "type": "unicode", "value": "user1" } + "username": { + "type": "unicode", + "value": "user1" + } } }, { "type": "User", "value": { - "username": { "type": "unicode", "value": "user2" } + "username": { + "type": "unicode", + "value": "user2" + } } } ] @@ -8389,14 +12308,18 @@ "type_stripped_response": { "items": [ { - "approvers": ["api_test_user"], + "approvers": [ + "api_test_user" + ], "email_cc_addresses": [], "email_message_id": "", "expiration_time_us": 2419244000000, "id": "approval:111111", "is_valid": false, "is_valid_message": "Need at least 1 additional approver for access.", - "notified_users": ["approver"], + "notified_users": [ + "approver" + ], "reason": "Running tests", "requestor": "api_test_user", "subject": { @@ -8405,10 +12328,12 @@ "build_time": "1980-01-01", "client_name": "GRR Monitor", "client_version": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "client_id": "C.1000000000000000", - "fleetspeak_enabled": false, "hardware_info": { "bios_version": "Bios-Version-0", "system_manufacturer": "System-Manufacturer-0" @@ -8416,7 +12341,10 @@ "interfaces": [ { "addresses": [ - { "address_type": "INET", "packed_bytes": "wKgAAA==" }, + { + "address_type": "INET", + "packed_bytes": "wKgAAA==" + }, { "address_type": "INET6", "packed_bytes": "IAGrzQAAAAAAAAAAAAAAAA==" @@ -8424,13 +12352,26 @@ ], "ifname": "if0" }, - { "ifname": "if1", "mac_address": "qrvM3e4A" }, - { "ifname": "if2", "mac_address": "u8zd7v8A" } + { + "ifname": "if1", + "mac_address": "qrvM3e4A" + }, + { + "ifname": "if2", + "mac_address": "u8zd7v8A" + } ], "knowledge_base": { "fqdn": "Host-0.example.com", "os": "Linux", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] }, "labels": [], "last_seen_at": 42000000, @@ -8442,7 +12383,14 @@ "version": "buster/sid" }, "urn": "aff4:/C.1000000000000000", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] } } ] @@ -8466,12 +12414,27 @@ "client_info": { "type": "ClientInformation", "value": { - "build_time": { "type": "unicode", "value": "1980-01-01" }, - "client_name": { "type": "unicode", "value": "GRR Monitor" }, - "client_version": { "type": "long", "value": 1234 }, + "build_time": { + "type": "unicode", + "value": "1980-01-01" + }, + "client_name": { + "type": "unicode", + "value": "GRR Monitor" + }, + "client_version": { + "type": "long", + "value": 1234 + }, "labels": [ - { "type": "unicode", "value": "label1" }, - { "type": "unicode", "value": "label2" } + { + "type": "unicode", + "value": "label1" + }, + { + "type": "unicode", + "value": "label2" + } ] } }, @@ -8479,12 +12442,18 @@ "type": "unicode", "value": "Client killed during transaction" }, - "crash_type": { "type": "unicode", "value": "Client Crash" }, + "crash_type": { + "type": "unicode", + "value": "Client Crash" + }, "session_id": { "type": "SessionID", "value": "" }, - "timestamp": { "type": "RDFDatetime", "value": 45000000 } + "timestamp": { + "type": "RDFDatetime", + "value": 45000000 + } } } ], @@ -8499,7 +12468,10 @@ "build_time": "1980-01-01", "client_name": "GRR Monitor", "client_version": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "crash_message": "Client killed during transaction", "crash_type": "Client Crash", @@ -8526,12 +12498,27 @@ "client_info": { "type": "ClientInformation", "value": { - "build_time": { "type": "unicode", "value": "1980-01-01" }, - "client_name": { "type": "unicode", "value": "GRR Monitor" }, - "client_version": { "type": "long", "value": 1234 }, + "build_time": { + "type": "unicode", + "value": "1980-01-01" + }, + "client_name": { + "type": "unicode", + "value": "GRR Monitor" + }, + "client_version": { + "type": "long", + "value": 1234 + }, "labels": [ - { "type": "unicode", "value": "label1" }, - { "type": "unicode", "value": "label2" } + { + "type": "unicode", + "value": "label1" + }, + { + "type": "unicode", + "value": "label2" + } ] } }, @@ -8539,12 +12526,18 @@ "type": "unicode", "value": "Client killed during transaction" }, - "crash_type": { "type": "unicode", "value": "Client Crash" }, + "crash_type": { + "type": "unicode", + "value": "Client Crash" + }, "session_id": { "type": "SessionID", "value": "" }, - "timestamp": { "type": "RDFDatetime", "value": 45000000 } + "timestamp": { + "type": "RDFDatetime", + "value": 45000000 + } } } ], @@ -8559,7 +12552,10 @@ "build_time": "1980-01-01", "client_name": "GRR Monitor", "client_version": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "crash_message": "Client killed during transaction", "crash_type": "Client Crash", @@ -8574,7 +12570,10 @@ { "api_method": "ListClientCrashes", "method": "GET", - "response": { "items": [], "total_count": 1 }, + "response": { + "items": [], + "total_count": 1 + }, "test_class": "ApiListClientCrashesHandlerRegressionTest_http_v1", "url": "/api/clients/C.1000000000000000/crashes?count=1&offset=1" } @@ -8587,17 +12586,34 @@ "items": [ { "type": "ClientLabel", - "value": { "name": { "type": "unicode", "value": "bar" } } + "value": { + "name": { + "type": "unicode", + "value": "bar" + } + } }, { "type": "ClientLabel", - "value": { "name": { "type": "unicode", "value": "foo" } } + "value": { + "name": { + "type": "unicode", + "value": "foo" + } + } } ] }, "test_class": "ApiListClientsLabelsHandlerRegressionTest_http_v1", "type_stripped_response": { - "items": [{ "name": "bar" }, { "name": "foo" }] + "items": [ + { + "name": "bar" + }, + { + "name": "foo" + } + ] }, "url": "/api/clients/labels" } @@ -8615,10 +12631,22 @@ "type": "ApiCronJobId", "value": "GRRVersionBreakDown" }, - "finished_at": { "type": "RDFDatetime", "value": 44000000 }, - "run_id": { "type": "ApiCronJobRunId", "value": "F:ABCDEF11" }, - "started_at": { "type": "RDFDatetime", "value": 44000000 }, - "status": { "type": "EnumNamedValue", "value": "FINISHED" } + "finished_at": { + "type": "RDFDatetime", + "value": 44000000 + }, + "run_id": { + "type": "ApiCronJobRunId", + "value": "F:ABCDEF11" + }, + "started_at": { + "type": "RDFDatetime", + "value": 44000000 + }, + "status": { + "type": "EnumNamedValue", + "value": "FINISHED" + } } } ] @@ -8647,7 +12675,10 @@ { "type": "ApiCronJob", "value": { - "allow_overruns": { "type": "bool", "value": false }, + "allow_overruns": { + "type": "bool", + "value": false + }, "args": { "type": "CronJobAction", "value": { @@ -8658,7 +12689,10 @@ "hunt_cron_action": { "type": "HuntCronAction", "value": { - "flow_args": { "type": "FileFinderArgs", "value": {} }, + "flow_args": { + "type": "FileFinderArgs", + "value": {} + }, "flow_name": { "type": "unicode", "value": "ClientFileFinder" @@ -8678,8 +12712,14 @@ "type": "long", "value": 1000 }, - "client_rate": { "type": "float", "value": 20.0 }, - "crash_limit": { "type": "long", "value": 100 } + "client_rate": { + "type": "float", + "value": 20.0 + }, + "crash_limit": { + "type": "long", + "value": 100 + } } } } @@ -8690,17 +12730,35 @@ "type": "ApiCronJobId", "value": "ClientFileFinder" }, - "description": { "type": "unicode", "value": "bar" }, - "enabled": { "type": "bool", "value": true }, - "frequency": { "type": "DurationSeconds", "value": 604800 }, - "is_failing": { "type": "bool", "value": false }, - "lifetime": { "type": "DurationSeconds", "value": 86400 } + "description": { + "type": "unicode", + "value": "bar" + }, + "enabled": { + "type": "bool", + "value": true + }, + "frequency": { + "type": "DurationSeconds", + "value": 604800 + }, + "is_failing": { + "type": "bool", + "value": false + }, + "lifetime": { + "type": "DurationSeconds", + "value": 86400 + } } }, { "type": "ApiCronJob", "value": { - "allow_overruns": { "type": "bool", "value": false }, + "allow_overruns": { + "type": "bool", + "value": false + }, "args": { "type": "CronJobAction", "value": { @@ -8711,8 +12769,14 @@ "hunt_cron_action": { "type": "HuntCronAction", "value": { - "flow_args": { "type": "FileFinderArgs", "value": {} }, - "flow_name": { "type": "unicode", "value": "FileFinder" }, + "flow_args": { + "type": "FileFinderArgs", + "value": {} + }, + "flow_name": { + "type": "unicode", + "value": "FileFinder" + }, "hunt_runner_args": { "type": "HuntRunnerArgs", "value": { @@ -8728,26 +12792,53 @@ "type": "long", "value": 1000 }, - "client_rate": { "type": "float", "value": 20.0 }, - "crash_limit": { "type": "long", "value": 100 } + "client_rate": { + "type": "float", + "value": 20.0 + }, + "crash_limit": { + "type": "long", + "value": 100 + } } } } } } }, - "cron_job_id": { "type": "ApiCronJobId", "value": "FileFinder" }, - "description": { "type": "unicode", "value": "foo" }, - "enabled": { "type": "bool", "value": false }, - "frequency": { "type": "DurationSeconds", "value": 86400 }, - "is_failing": { "type": "bool", "value": false }, - "lifetime": { "type": "DurationSeconds", "value": 7200 } + "cron_job_id": { + "type": "ApiCronJobId", + "value": "FileFinder" + }, + "description": { + "type": "unicode", + "value": "foo" + }, + "enabled": { + "type": "bool", + "value": false + }, + "frequency": { + "type": "DurationSeconds", + "value": 86400 + }, + "is_failing": { + "type": "bool", + "value": false + }, + "lifetime": { + "type": "DurationSeconds", + "value": 7200 + } } }, { "type": "ApiCronJob", "value": { - "allow_overruns": { "type": "bool", "value": false }, + "allow_overruns": { + "type": "bool", + "value": false + }, "args": { "type": "CronJobAction", "value": { @@ -8758,7 +12849,10 @@ "hunt_cron_action": { "type": "HuntCronAction", "value": { - "flow_args": { "type": "ListDirectoryArgs", "value": {} }, + "flow_args": { + "type": "ListDirectoryArgs", + "value": {} + }, "flow_name": { "type": "unicode", "value": "ListDirectory" @@ -8778,8 +12872,14 @@ "type": "long", "value": 1000 }, - "client_rate": { "type": "float", "value": 20.0 }, - "crash_limit": { "type": "long", "value": 100 } + "client_rate": { + "type": "float", + "value": 20.0 + }, + "crash_limit": { + "type": "long", + "value": 100 + } } } } @@ -8790,13 +12890,34 @@ "type": "ApiCronJobId", "value": "ListDirectory" }, - "description": { "type": "unicode", "value": "" }, - "enabled": { "type": "bool", "value": true }, - "frequency": { "type": "DurationSeconds", "value": 604800 }, - "is_failing": { "type": "bool", "value": true }, - "last_run_status": { "type": "EnumNamedValue", "value": "ERROR" }, - "last_run_time": { "type": "RDFDatetime", "value": 230000000 }, - "lifetime": { "type": "DurationSeconds", "value": 86400 } + "description": { + "type": "unicode", + "value": "" + }, + "enabled": { + "type": "bool", + "value": true + }, + "frequency": { + "type": "DurationSeconds", + "value": 604800 + }, + "is_failing": { + "type": "bool", + "value": true + }, + "last_run_status": { + "type": "EnumNamedValue", + "value": "ERROR" + }, + "last_run_time": { + "type": "RDFDatetime", + "value": 230000000 + }, + "lifetime": { + "type": "DurationSeconds", + "value": 86400 + } } } ], @@ -8891,23 +13012,59 @@ { "type": "ApiFile", "value": { - "age": { "type": "RDFDatetime", "value": 42000000 }, - "is_directory": { "type": "bool", "value": false }, - "last_collected": { "type": "RDFDatetime", "value": 42000000 }, - "last_collected_size": { "type": "long", "value": 11 }, - "name": { "type": "unicode", "value": "bash" }, - "path": { "type": "unicode", "value": "fs/tsk/c/bin/bash" } + "age": { + "type": "RDFDatetime", + "value": 42000000 + }, + "is_directory": { + "type": "bool", + "value": false + }, + "last_collected": { + "type": "RDFDatetime", + "value": 42000000 + }, + "last_collected_size": { + "type": "long", + "value": 11 + }, + "name": { + "type": "unicode", + "value": "bash" + }, + "path": { + "type": "unicode", + "value": "fs/tsk/c/bin/bash" + } } }, { "type": "ApiFile", "value": { - "age": { "type": "RDFDatetime", "value": 42000000 }, - "is_directory": { "type": "bool", "value": false }, - "last_collected": { "type": "RDFDatetime", "value": 42000000 }, - "last_collected_size": { "type": "long", "value": 11 }, - "name": { "type": "unicode", "value": "rbash" }, - "path": { "type": "unicode", "value": "fs/tsk/c/bin/rbash" } + "age": { + "type": "RDFDatetime", + "value": 42000000 + }, + "is_directory": { + "type": "bool", + "value": false + }, + "last_collected": { + "type": "RDFDatetime", + "value": 42000000 + }, + "last_collected_size": { + "type": "long", + "value": 11 + }, + "name": { + "type": "unicode", + "value": "rbash" + }, + "path": { + "type": "unicode", + "value": "fs/tsk/c/bin/rbash" + } } } ] @@ -8943,23 +13100,59 @@ { "type": "ApiFile", "value": { - "age": { "type": "RDFDatetime", "value": 42000000 }, - "is_directory": { "type": "bool", "value": false }, - "last_collected": { "type": "RDFDatetime", "value": 42000000 }, - "last_collected_size": { "type": "long", "value": 11 }, - "name": { "type": "unicode", "value": "bash" }, - "path": { "type": "unicode", "value": "fs/tsk/c/bin/bash" } + "age": { + "type": "RDFDatetime", + "value": 42000000 + }, + "is_directory": { + "type": "bool", + "value": false + }, + "last_collected": { + "type": "RDFDatetime", + "value": 42000000 + }, + "last_collected_size": { + "type": "long", + "value": 11 + }, + "name": { + "type": "unicode", + "value": "bash" + }, + "path": { + "type": "unicode", + "value": "fs/tsk/c/bin/bash" + } } }, { "type": "ApiFile", "value": { - "age": { "type": "RDFDatetime", "value": 42000000 }, - "is_directory": { "type": "bool", "value": false }, - "last_collected": { "type": "RDFDatetime", "value": 42000000 }, - "last_collected_size": { "type": "long", "value": 11 }, - "name": { "type": "unicode", "value": "rbash" }, - "path": { "type": "unicode", "value": "fs/tsk/c/bin/rbash" } + "age": { + "type": "RDFDatetime", + "value": 42000000 + }, + "is_directory": { + "type": "bool", + "value": false + }, + "last_collected": { + "type": "RDFDatetime", + "value": 42000000 + }, + "last_collected_size": { + "type": "long", + "value": 11 + }, + "name": { + "type": "unicode", + "value": "rbash" + }, + "path": { + "type": "unicode", + "value": "fs/tsk/c/bin/rbash" + } } } ] @@ -8997,36 +13190,75 @@ { "type": "ApiFlowDescriptor", "value": { - "args_type": { "type": "unicode", "value": "FileFinderArgs" }, + "args_type": { + "type": "unicode", + "value": "FileFinderArgs" + }, "behaviours": [ - { "type": "unicode", "value": "ADVANCED" }, - { "type": "unicode", "value": "BASIC" } + { + "type": "unicode", + "value": "ADVANCED" + }, + { + "type": "unicode", + "value": "BASIC" + } ], - "category": { "type": "unicode", "value": "Filesystem" }, - "default_args": { "type": "FileFinderArgs", "value": {} }, + "category": { + "type": "unicode", + "value": "Filesystem" + }, + "default_args": { + "type": "FileFinderArgs", + "value": {} + }, "doc": { "type": "unicode", "value": "This flow looks for files matching given criteria and acts on them.\n\n FileFinder searches for files that match glob expressions. The \"action\"\n (e.g. Download) is applied to files that match all given \"conditions\".\n Matches are then written to the results collection. If there are no\n \"conditions\" specified, \"action\" is just applied to all found files.\n \n\n Call Spec:\n flow.StartFlow(client_id=client_id, flow_cls=file_finder.FileFinder, paths=paths, pathtype=pathtype, conditions=conditions, action=action, process_non_regular_files=process_non_regular_files, follow_links=follow_links, xdev=xdev, implementation_type=implementation_type)\n\n Args:\n action\n description: \n type: FileFinderAction\n default: None\n\n conditions\n description: These conditions will be applied to all files that match the path arguments.\n type: \n default: None\n\n follow_links\n description: Should symbolic links be followed in recursive directory listings.\n type: bool\n default: False\n\n implementation_type\n description: Force use of an implementation.\n type: EnumNamedValue\n default: 0\n\n paths\n description: A path to glob that can contain %% expansions.\n type: \n default: None\n\n pathtype\n description: Path type to glob in.\n type: EnumNamedValue\n default: OS\n\n process_non_regular_files\n description: Look both into regular files and non-regular files (devices, named pipes, sockets). NOTE: This is very dangerous and should be used with care.\n type: bool\n default: 0\n\n xdev\n description: Behavior when ecountering device boundaries while doing recursive searches.\n type: EnumNamedValue\n default: LOCAL\n" }, - "friendly_name": { "type": "unicode", "value": "File Finder" }, - "name": { "type": "unicode", "value": "FileFinder" } + "friendly_name": { + "type": "unicode", + "value": "File Finder" + }, + "name": { + "type": "unicode", + "value": "FileFinder" + } } }, { "type": "ApiFlowDescriptor", "value": { - "args_type": { "type": "unicode", "value": "ListProcessesArgs" }, + "args_type": { + "type": "unicode", + "value": "ListProcessesArgs" + }, "behaviours": [ - { "type": "unicode", "value": "ADVANCED" }, - { "type": "unicode", "value": "BASIC" } + { + "type": "unicode", + "value": "ADVANCED" + }, + { + "type": "unicode", + "value": "BASIC" + } ], - "category": { "type": "unicode", "value": "Processes" }, - "default_args": { "type": "ListProcessesArgs", "value": {} }, + "category": { + "type": "unicode", + "value": "Processes" + }, + "default_args": { + "type": "ListProcessesArgs", + "value": {} + }, "doc": { "type": "unicode", "value": "List running processes on a system.\n\n Call Spec:\n flow.StartFlow(client_id=client_id, flow_cls=processes.ListProcesses, filename_regex=filename_regex, fetch_binaries=fetch_binaries, connection_states=connection_states, pids=pids)\n\n Args:\n connection_states\n description: Network connection states to match. If a process has any network connections in any status listed here, it will be considered a match\n type: \n default: None\n\n fetch_binaries\n description: \n type: bool\n default: False\n\n filename_regex\n description: Regex used to filter the list of processes. The regex is applied to the full path and not only to the filename.\n type: RegularExpression\n default: .\n\n pids\n description: \n type: \n default: None\n" }, - "name": { "type": "unicode", "value": "ListProcesses" } + "name": { + "type": "unicode", + "value": "ListProcesses" + } } } ] @@ -9036,7 +13268,10 @@ "items": [ { "args_type": "FileFinderArgs", - "behaviours": ["ADVANCED", "BASIC"], + "behaviours": [ + "ADVANCED", + "BASIC" + ], "category": "Filesystem", "default_args": {}, "doc": "This flow looks for files matching given criteria and acts on them.\n\n FileFinder searches for files that match glob expressions. The \"action\"\n (e.g. Download) is applied to files that match all given \"conditions\".\n Matches are then written to the results collection. If there are no\n \"conditions\" specified, \"action\" is just applied to all found files.\n \n\n Call Spec:\n flow.StartFlow(client_id=client_id, flow_cls=file_finder.FileFinder, paths=paths, pathtype=pathtype, conditions=conditions, action=action, process_non_regular_files=process_non_regular_files, follow_links=follow_links, xdev=xdev, implementation_type=implementation_type)\n\n Args:\n action\n description: \n type: FileFinderAction\n default: None\n\n conditions\n description: These conditions will be applied to all files that match the path arguments.\n type: \n default: None\n\n follow_links\n description: Should symbolic links be followed in recursive directory listings.\n type: bool\n default: False\n\n implementation_type\n description: Force use of an implementation.\n type: EnumNamedValue\n default: 0\n\n paths\n description: A path to glob that can contain %% expansions.\n type: \n default: None\n\n pathtype\n description: Path type to glob in.\n type: EnumNamedValue\n default: OS\n\n process_non_regular_files\n description: Look both into regular files and non-regular files (devices, named pipes, sockets). NOTE: This is very dangerous and should be used with care.\n type: bool\n default: 0\n\n xdev\n description: Behavior when ecountering device boundaries while doing recursive searches.\n type: EnumNamedValue\n default: LOCAL\n", @@ -9045,7 +13280,10 @@ }, { "args_type": "ListProcessesArgs", - "behaviours": ["ADVANCED", "BASIC"], + "behaviours": [ + "ADVANCED", + "BASIC" + ], "category": "Processes", "default_args": {}, "doc": "List running processes on a system.\n\n Call Spec:\n flow.StartFlow(client_id=client_id, flow_cls=processes.ListProcesses, filename_regex=filename_regex, fetch_binaries=fetch_binaries, connection_states=connection_states, pids=pids)\n\n Args:\n connection_states\n description: Network connection states to match. If a process has any network connections in any status listed here, it will be considered a match\n type: \n default: None\n\n fetch_binaries\n description: \n type: bool\n default: False\n\n filename_regex\n description: Regex used to filter the list of processes. The regex is applied to the full path and not only to the filename.\n type: RegularExpression\n default: .\n\n pids\n description: \n type: \n default: None\n", @@ -9065,23 +13303,35 @@ { "type": "ApiFlowLog", "value": { - "flow_id": { "type": "ApiFlowId", "value": "W:ABCDEF" }, + "flow_id": { + "type": "ApiFlowId", + "value": "W:ABCDEF" + }, "log_message": { "type": "unicode", "value": "Sample message: foo." }, - "timestamp": { "type": "RDFDatetime", "value": 52000000 } - } + "timestamp": { + "type": "RDFDatetime", + "value": 52000000 + } + } }, { "type": "ApiFlowLog", "value": { - "flow_id": { "type": "ApiFlowId", "value": "W:ABCDEF" }, + "flow_id": { + "type": "ApiFlowId", + "value": "W:ABCDEF" + }, "log_message": { "type": "unicode", "value": "Sample message: bar." }, - "timestamp": { "type": "RDFDatetime", "value": 55000000 } + "timestamp": { + "type": "RDFDatetime", + "value": 55000000 + } } } ], @@ -9113,12 +13363,18 @@ { "type": "ApiFlowLog", "value": { - "flow_id": { "type": "ApiFlowId", "value": "W:ABCDEF" }, + "flow_id": { + "type": "ApiFlowId", + "value": "W:ABCDEF" + }, "log_message": { "type": "unicode", "value": "Sample message: foo." }, - "timestamp": { "type": "RDFDatetime", "value": 52000000 } + "timestamp": { + "type": "RDFDatetime", + "value": 52000000 + } } } ], @@ -9145,12 +13401,18 @@ { "type": "ApiFlowLog", "value": { - "flow_id": { "type": "ApiFlowId", "value": "W:ABCDEF" }, + "flow_id": { + "type": "ApiFlowId", + "value": "W:ABCDEF" + }, "log_message": { "type": "unicode", "value": "Sample message: bar." }, - "timestamp": { "type": "RDFDatetime", "value": 55000000 } + "timestamp": { + "type": "RDFDatetime", + "value": 55000000 + } } } ], @@ -9179,9 +13441,18 @@ { "type": "OutputPluginBatchProcessingStatus", "value": { - "batch_index": { "type": "long", "value": 0 }, - "batch_size": { "type": "long", "value": 0 }, - "status": { "type": "EnumNamedValue", "value": "ERROR" }, + "batch_index": { + "type": "long", + "value": 0 + }, + "batch_size": { + "type": "long", + "value": 0 + }, + "status": { + "type": "EnumNamedValue", + "value": "ERROR" + }, "summary": { "type": "unicode", "value": "Error while processing 1 replies: Oh no!" @@ -9215,10 +13486,22 @@ { "type": "OutputPluginBatchProcessingStatus", "value": { - "batch_index": { "type": "long", "value": 0 }, - "batch_size": { "type": "long", "value": 0 }, - "status": { "type": "EnumNamedValue", "value": "SUCCESS" }, - "summary": { "type": "unicode", "value": "Processed 1 replies." } + "batch_index": { + "type": "long", + "value": 0 + }, + "batch_size": { + "type": "long", + "value": 0 + }, + "status": { + "type": "EnumNamedValue", + "value": "SUCCESS" + }, + "summary": { + "type": "unicode", + "value": "Processed 1 replies." + } } } ], @@ -9248,7 +13531,10 @@ { "type": "ApiOutputPlugin", "value": { - "id": { "type": "unicode", "value": "EmailOutputPlugin_0" }, + "id": { + "type": "unicode", + "value": "EmailOutputPlugin_0" + }, "plugin_descriptor": { "type": "OutputPluginDescriptor", "value": { @@ -9259,7 +13545,10 @@ "type": "DomainEmailAddress", "value": "test@localhost" }, - "emails_limit": { "type": "long", "value": 42 } + "emails_limit": { + "type": "long", + "value": 42 + } } }, "plugin_name": { @@ -9278,10 +13567,16 @@ "type": "DomainEmailAddress", "value": "test@localhost" }, - "emails_limit": { "type": "long", "value": 42 } + "emails_limit": { + "type": "long", + "value": 42 + } } }, - "emails_sent": { "type": "long", "value": 0 }, + "emails_sent": { + "type": "long", + "value": 0 + }, "errors": [], "logs": [] } @@ -9296,11 +13591,17 @@ { "id": "EmailOutputPlugin_0", "plugin_descriptor": { - "args": { "email_address": "test@localhost", "emails_limit": 42 }, + "args": { + "email_address": "test@localhost", + "emails_limit": 42 + }, "plugin_name": "EmailOutputPlugin" }, "state": { - "args": { "email_address": "test@localhost", "emails_limit": 42 }, + "args": { + "email_address": "test@localhost", + "emails_limit": 42 + }, "emails_sent": 0, "errors": [], "logs": [] @@ -9320,7 +13621,10 @@ { "type": "ApiFlowRequest", "value": { - "request_id": { "type": "unicode", "value": "1" }, + "request_id": { + "type": "unicode", + "value": "1" + }, "request_state": { "type": "RequestState", "value": { @@ -9328,7 +13632,10 @@ "type": "ClientURN", "value": "aff4:/C.1000000000000000" }, - "id": { "type": "long", "value": 1 }, + "id": { + "type": "long", + "value": 1 + }, "next_state": { "type": "unicode", "value": "IterateProcesses" @@ -9343,27 +13650,51 @@ { "type": "GrrMessage", "value": { - "request_id": { "type": "long", "value": 1 }, - "response_id": { "type": "long", "value": 1 }, + "request_id": { + "type": "long", + "value": 1 + }, + "response_id": { + "type": "long", + "value": 1 + }, "session_id": { "type": "FlowSessionID", "value": "aff4:/C.1000000000000000/flows/W:ABCDEF" }, - "timestamp": { "type": "RDFDatetime", "value": 42000000 }, - "type": { "type": "EnumNamedValue", "value": "MESSAGE" } + "timestamp": { + "type": "RDFDatetime", + "value": 42000000 + }, + "type": { + "type": "EnumNamedValue", + "value": "MESSAGE" + } } }, { "type": "GrrMessage", "value": { - "request_id": { "type": "long", "value": 1 }, - "response_id": { "type": "long", "value": 2 }, + "request_id": { + "type": "long", + "value": 1 + }, + "response_id": { + "type": "long", + "value": 2 + }, "session_id": { "type": "FlowSessionID", "value": "aff4:/C.1000000000000000/flows/W:ABCDEF" }, - "timestamp": { "type": "RDFDatetime", "value": 42000000 }, - "type": { "type": "EnumNamedValue", "value": "STATUS" } + "timestamp": { + "type": "RDFDatetime", + "value": 42000000 + }, + "type": { + "type": "EnumNamedValue", + "value": "STATUS" + } } } ] @@ -9419,8 +13750,14 @@ "pathspec": { "type": "PathSpec", "value": { - "path": { "type": "unicode", "value": "/tmp/evil.txt" }, - "pathtype": { "type": "EnumNamedValue", "value": "OS" } + "path": { + "type": "unicode", + "value": "/tmp/evil.txt" + }, + "pathtype": { + "type": "EnumNamedValue", + "value": "OS" + } } }, "st_atime": { @@ -9431,21 +13768,48 @@ "type": "RDFDatetimeSeconds", "value": 1336129892 }, - "st_dev": { "type": "long", "value": 64512 }, - "st_gid": { "type": "long", "value": 5000 }, - "st_ino": { "type": "long", "value": 1063090 }, - "st_mode": { "type": "StatMode", "value": 33184 }, + "st_dev": { + "type": "long", + "value": 64512 + }, + "st_gid": { + "type": "long", + "value": 5000 + }, + "st_ino": { + "type": "long", + "value": 1063090 + }, + "st_mode": { + "type": "StatMode", + "value": 33184 + }, "st_mtime": { "type": "RDFDatetimeSeconds", "value": 1336129892 }, - "st_nlink": { "type": "long", "value": 1 }, - "st_size": { "type": "long", "value": 12 }, - "st_uid": { "type": "long", "value": 139592 } + "st_nlink": { + "type": "long", + "value": 1 + }, + "st_size": { + "type": "long", + "value": 12 + }, + "st_uid": { + "type": "long", + "value": 139592 + } } }, - "payload_type": { "type": "unicode", "value": "StatEntry" }, - "timestamp": { "type": "RDFDatetime", "value": 42000000 } + "payload_type": { + "type": "unicode", + "value": "StatEntry" + }, + "timestamp": { + "type": "RDFDatetime", + "value": 42000000 + } } } ] @@ -9455,7 +13819,10 @@ "items": [ { "payload": { - "pathspec": { "path": "/tmp/evil.txt", "pathtype": "OS" }, + "pathspec": { + "path": "/tmp/evil.txt", + "pathtype": "OS" + }, "st_atime": 1336469177, "st_ctime": 1336129892, "st_dev": 64512, @@ -9477,7 +13844,9 @@ { "api_method": "ListFlowResults", "method": "GET", - "response": { "items": [] }, + "response": { + "items": [] + }, "test_class": "ApiListFlowResultsHandlerRegressionTest_http_v1", "url": "/api/clients/C.1000000000000000/flows/W:ABCDEF/results?filter=benign" } @@ -9495,11 +13864,26 @@ "type": "ApiClientId", "value": "C.1000000000000000" }, - "creator": { "type": "unicode", "value": "GRRWorker" }, - "flow_id": { "type": "ApiFlowId", "value": "F:ABCDEF11" }, - "is_robot": { "type": "bool", "value": true }, - "last_active_at": { "type": "RDFDatetime", "value": 44000000 }, - "name": { "type": "unicode", "value": "ListProcesses" }, + "creator": { + "type": "unicode", + "value": "GRRWorker" + }, + "flow_id": { + "type": "ApiFlowId", + "value": "F:ABCDEF11" + }, + "is_robot": { + "type": "bool", + "value": true + }, + "last_active_at": { + "type": "RDFDatetime", + "value": 44000000 + }, + "name": { + "type": "unicode", + "value": "ListProcesses" + }, "nested_flows": [], "runner_args": { "type": "FlowRunnerArgs", @@ -9508,12 +13892,24 @@ "type": "ClientURN", "value": "aff4:/C.1000000000000000" }, - "flow_name": { "type": "unicode", "value": "ListProcesses" }, - "notify_to_user": { "type": "bool", "value": false } + "flow_name": { + "type": "unicode", + "value": "ListProcesses" + }, + "notify_to_user": { + "type": "bool", + "value": false + } } }, - "started_at": { "type": "RDFDatetime", "value": 44000000 }, - "state": { "type": "EnumNamedValue", "value": "RUNNING" }, + "started_at": { + "type": "RDFDatetime", + "value": 44000000 + }, + "state": { + "type": "EnumNamedValue", + "value": "RUNNING" + }, "urn": { "type": "SessionID", "value": "aff4:/C.1000000000000000/flows/F:ABCDEF11" @@ -9527,11 +13923,26 @@ "type": "ApiClientId", "value": "C.1000000000000000" }, - "creator": { "type": "unicode", "value": "api_test_user" }, - "flow_id": { "type": "ApiFlowId", "value": "F:ABCDEF10" }, - "is_robot": { "type": "bool", "value": false }, - "last_active_at": { "type": "RDFDatetime", "value": 43000000 }, - "name": { "type": "unicode", "value": "Interrogate" }, + "creator": { + "type": "unicode", + "value": "api_test_user" + }, + "flow_id": { + "type": "ApiFlowId", + "value": "F:ABCDEF10" + }, + "is_robot": { + "type": "bool", + "value": false + }, + "last_active_at": { + "type": "RDFDatetime", + "value": 43000000 + }, + "name": { + "type": "unicode", + "value": "Interrogate" + }, "nested_flows": [], "runner_args": { "type": "FlowRunnerArgs", @@ -9540,12 +13951,24 @@ "type": "ClientURN", "value": "aff4:/C.1000000000000000" }, - "flow_name": { "type": "unicode", "value": "Interrogate" }, - "notify_to_user": { "type": "bool", "value": true } + "flow_name": { + "type": "unicode", + "value": "Interrogate" + }, + "notify_to_user": { + "type": "bool", + "value": true + } } }, - "started_at": { "type": "RDFDatetime", "value": 43000000 }, - "state": { "type": "EnumNamedValue", "value": "RUNNING" }, + "started_at": { + "type": "RDFDatetime", + "value": 43000000 + }, + "state": { + "type": "EnumNamedValue", + "value": "RUNNING" + }, "urn": { "type": "SessionID", "value": "aff4:/C.1000000000000000/flows/F:ABCDEF10" @@ -9603,21 +14026,45 @@ { "type": "ApiFlow", "value": { - "args": { "type": "ListProcessesArgs", "value": {} }, + "args": { + "type": "ListProcessesArgs", + "value": {} + }, "client_id": { "type": "ApiClientId", "value": "C.1000000000000000" }, - "creator": { "type": "unicode", "value": "GRRWorker" }, - "flow_id": { "type": "ApiFlowId", "value": "F:ABCDEF11" }, - "is_robot": { "type": "bool", "value": true }, - "last_active_at": { "type": "RDFDatetime", "value": 44000000 }, - "name": { "type": "unicode", "value": "ListProcesses" }, - "progress": { "type": "DefaultFlowProgress", "value": {} }, + "creator": { + "type": "unicode", + "value": "GRRWorker" + }, + "flow_id": { + "type": "ApiFlowId", + "value": "F:ABCDEF11" + }, + "is_robot": { + "type": "bool", + "value": true + }, + "last_active_at": { + "type": "RDFDatetime", + "value": 44000000 + }, + "name": { + "type": "unicode", + "value": "ListProcesses" + }, + "progress": { + "type": "DefaultFlowProgress", + "value": {} + }, "result_metadata": { "type": "FlowResultMetadata", "value": { - "is_metadata_set": { "type": "bool", "value": true } + "is_metadata_set": { + "type": "bool", + "value": true + } } }, "runner_args": { @@ -9627,12 +14074,24 @@ "type": "ClientURN", "value": "aff4:/C.1000000000000000" }, - "flow_name": { "type": "unicode", "value": "ListProcesses" }, - "notify_to_user": { "type": "bool", "value": false } + "flow_name": { + "type": "unicode", + "value": "ListProcesses" + }, + "notify_to_user": { + "type": "bool", + "value": false + } } }, - "started_at": { "type": "RDFDatetime", "value": 44000000 }, - "state": { "type": "EnumNamedValue", "value": "RUNNING" }, + "started_at": { + "type": "RDFDatetime", + "value": 44000000 + }, + "state": { + "type": "EnumNamedValue", + "value": "RUNNING" + }, "urn": { "type": "SessionID", "value": "aff4:/C.1000000000000000/flows/F:ABCDEF11" @@ -9642,21 +14101,45 @@ { "type": "ApiFlow", "value": { - "args": { "type": "InterrogateArgs", "value": {} }, + "args": { + "type": "InterrogateArgs", + "value": {} + }, "client_id": { "type": "ApiClientId", "value": "C.1000000000000000" }, - "creator": { "type": "unicode", "value": "api_test_user" }, - "flow_id": { "type": "ApiFlowId", "value": "F:ABCDEF10" }, - "is_robot": { "type": "bool", "value": false }, - "last_active_at": { "type": "RDFDatetime", "value": 43000000 }, - "name": { "type": "unicode", "value": "Interrogate" }, - "progress": { "type": "DefaultFlowProgress", "value": {} }, + "creator": { + "type": "unicode", + "value": "api_test_user" + }, + "flow_id": { + "type": "ApiFlowId", + "value": "F:ABCDEF10" + }, + "is_robot": { + "type": "bool", + "value": false + }, + "last_active_at": { + "type": "RDFDatetime", + "value": 43000000 + }, + "name": { + "type": "unicode", + "value": "Interrogate" + }, + "progress": { + "type": "DefaultFlowProgress", + "value": {} + }, "result_metadata": { "type": "FlowResultMetadata", "value": { - "is_metadata_set": { "type": "bool", "value": true } + "is_metadata_set": { + "type": "bool", + "value": true + } } }, "runner_args": { @@ -9666,12 +14149,24 @@ "type": "ClientURN", "value": "aff4:/C.1000000000000000" }, - "flow_name": { "type": "unicode", "value": "Interrogate" }, - "notify_to_user": { "type": "bool", "value": true } + "flow_name": { + "type": "unicode", + "value": "Interrogate" + }, + "notify_to_user": { + "type": "bool", + "value": true + } } }, - "started_at": { "type": "RDFDatetime", "value": 43000000 }, - "state": { "type": "EnumNamedValue", "value": "RUNNING" }, + "started_at": { + "type": "RDFDatetime", + "value": 43000000 + }, + "state": { + "type": "EnumNamedValue", + "value": "RUNNING" + }, "urn": { "type": "SessionID", "value": "aff4:/C.1000000000000000/flows/F:ABCDEF10" @@ -9692,7 +14187,9 @@ "last_active_at": 44000000, "name": "ListProcesses", "progress": {}, - "result_metadata": { "is_metadata_set": true }, + "result_metadata": { + "is_metadata_set": true + }, "runner_args": { "client_id": "aff4:/C.1000000000000000", "flow_name": "ListProcesses", @@ -9711,7 +14208,9 @@ "last_active_at": 43000000, "name": "Interrogate", "progress": {}, - "result_metadata": { "is_metadata_set": true }, + "result_metadata": { + "is_metadata_set": true + }, "runner_args": { "client_id": "aff4:/C.1000000000000000", "flow_name": "Interrogate", @@ -9733,21 +14232,45 @@ { "type": "ApiFlow", "value": { - "args": { "type": "ListProcessesArgs", "value": {} }, + "args": { + "type": "ListProcessesArgs", + "value": {} + }, "client_id": { "type": "ApiClientId", "value": "C.1000000000000000" }, - "creator": { "type": "unicode", "value": "GRRWorker" }, - "flow_id": { "type": "ApiFlowId", "value": "F:ABCDEF11" }, - "is_robot": { "type": "bool", "value": true }, - "last_active_at": { "type": "RDFDatetime", "value": 44000000 }, - "name": { "type": "unicode", "value": "ListProcesses" }, - "progress": { "type": "DefaultFlowProgress", "value": {} }, + "creator": { + "type": "unicode", + "value": "GRRWorker" + }, + "flow_id": { + "type": "ApiFlowId", + "value": "F:ABCDEF11" + }, + "is_robot": { + "type": "bool", + "value": true + }, + "last_active_at": { + "type": "RDFDatetime", + "value": 44000000 + }, + "name": { + "type": "unicode", + "value": "ListProcesses" + }, + "progress": { + "type": "DefaultFlowProgress", + "value": {} + }, "result_metadata": { "type": "FlowResultMetadata", "value": { - "is_metadata_set": { "type": "bool", "value": true } + "is_metadata_set": { + "type": "bool", + "value": true + } } }, "runner_args": { @@ -9757,12 +14280,24 @@ "type": "ClientURN", "value": "aff4:/C.1000000000000000" }, - "flow_name": { "type": "unicode", "value": "ListProcesses" }, - "notify_to_user": { "type": "bool", "value": false } + "flow_name": { + "type": "unicode", + "value": "ListProcesses" + }, + "notify_to_user": { + "type": "bool", + "value": false + } } }, - "started_at": { "type": "RDFDatetime", "value": 44000000 }, - "state": { "type": "EnumNamedValue", "value": "RUNNING" }, + "started_at": { + "type": "RDFDatetime", + "value": 44000000 + }, + "state": { + "type": "EnumNamedValue", + "value": "RUNNING" + }, "urn": { "type": "SessionID", "value": "aff4:/C.1000000000000000/flows/F:ABCDEF11" @@ -9783,7 +14318,9 @@ "last_active_at": 44000000, "name": "ListProcesses", "progress": {}, - "result_metadata": { "is_metadata_set": true }, + "result_metadata": { + "is_metadata_set": true + }, "runner_args": { "client_id": "aff4:/C.1000000000000000", "flow_name": "ListProcesses", @@ -9805,21 +14342,45 @@ { "type": "ApiFlow", "value": { - "args": { "type": "InterrogateArgs", "value": {} }, + "args": { + "type": "InterrogateArgs", + "value": {} + }, "client_id": { "type": "ApiClientId", "value": "C.1000000000000000" }, - "creator": { "type": "unicode", "value": "api_test_user" }, - "flow_id": { "type": "ApiFlowId", "value": "F:ABCDEF10" }, - "is_robot": { "type": "bool", "value": false }, - "last_active_at": { "type": "RDFDatetime", "value": 43000000 }, - "name": { "type": "unicode", "value": "Interrogate" }, - "progress": { "type": "DefaultFlowProgress", "value": {} }, + "creator": { + "type": "unicode", + "value": "api_test_user" + }, + "flow_id": { + "type": "ApiFlowId", + "value": "F:ABCDEF10" + }, + "is_robot": { + "type": "bool", + "value": false + }, + "last_active_at": { + "type": "RDFDatetime", + "value": 43000000 + }, + "name": { + "type": "unicode", + "value": "Interrogate" + }, + "progress": { + "type": "DefaultFlowProgress", + "value": {} + }, "result_metadata": { "type": "FlowResultMetadata", "value": { - "is_metadata_set": { "type": "bool", "value": true } + "is_metadata_set": { + "type": "bool", + "value": true + } } }, "runner_args": { @@ -9829,12 +14390,24 @@ "type": "ClientURN", "value": "aff4:/C.1000000000000000" }, - "flow_name": { "type": "unicode", "value": "Interrogate" }, - "notify_to_user": { "type": "bool", "value": true } + "flow_name": { + "type": "unicode", + "value": "Interrogate" + }, + "notify_to_user": { + "type": "bool", + "value": true + } } }, - "started_at": { "type": "RDFDatetime", "value": 43000000 }, - "state": { "type": "EnumNamedValue", "value": "RUNNING" }, + "started_at": { + "type": "RDFDatetime", + "value": 43000000 + }, + "state": { + "type": "EnumNamedValue", + "value": "RUNNING" + }, "urn": { "type": "SessionID", "value": "aff4:/C.1000000000000000/flows/F:ABCDEF10" @@ -9855,7 +14428,9 @@ "last_active_at": 43000000, "name": "Interrogate", "progress": {}, - "result_metadata": { "is_metadata_set": true }, + "result_metadata": { + "is_metadata_set": true + }, "runner_args": { "client_id": "aff4:/C.1000000000000000", "flow_name": "Interrogate", @@ -9881,11 +14456,26 @@ "type": "ApiClientId", "value": "C.1000000000000000" }, - "creator": { "type": "unicode", "value": "api_test_user" }, - "flow_id": { "type": "ApiFlowId", "value": "F:ABCDEF10" }, - "is_robot": { "type": "bool", "value": false }, - "last_active_at": { "type": "RDFDatetime", "value": 43000000 }, - "name": { "type": "unicode", "value": "Interrogate" }, + "creator": { + "type": "unicode", + "value": "api_test_user" + }, + "flow_id": { + "type": "ApiFlowId", + "value": "F:ABCDEF10" + }, + "is_robot": { + "type": "bool", + "value": false + }, + "last_active_at": { + "type": "RDFDatetime", + "value": 43000000 + }, + "name": { + "type": "unicode", + "value": "Interrogate" + }, "nested_flows": [], "runner_args": { "type": "FlowRunnerArgs", @@ -9894,12 +14484,24 @@ "type": "ClientURN", "value": "aff4:/C.1000000000000000" }, - "flow_name": { "type": "unicode", "value": "Interrogate" }, - "notify_to_user": { "type": "bool", "value": true } + "flow_name": { + "type": "unicode", + "value": "Interrogate" + }, + "notify_to_user": { + "type": "bool", + "value": true + } } }, - "started_at": { "type": "RDFDatetime", "value": 43000000 }, - "state": { "type": "EnumNamedValue", "value": "RUNNING" }, + "started_at": { + "type": "RDFDatetime", + "value": 43000000 + }, + "state": { + "type": "EnumNamedValue", + "value": "RUNNING" + }, "urn": { "type": "SessionID", "value": "aff4:/C.1000000000000000/flows/F:ABCDEF10" @@ -9942,29 +14544,59 @@ { "type": "ApiGrrBinary", "value": { - "has_valid_signature": { "type": "bool", "value": true }, - "path": { "type": "unicode", "value": "windows/test.exe" }, - "size": { "type": "ByteSize", "value": 18 }, - "timestamp": { "type": "RDFDatetime", "value": 42000000 }, - "type": { "type": "EnumNamedValue", "value": "EXECUTABLE" } + "has_valid_signature": { + "type": "bool", + "value": true + }, + "path": { + "type": "unicode", + "value": "windows/test.exe" + }, + "size": { + "type": "ByteSize", + "value": 18 + }, + "timestamp": { + "type": "RDFDatetime", + "value": 42000000 + }, + "type": { + "type": "EnumNamedValue", + "value": "EXECUTABLE" + } } }, { "type": "ApiGrrBinary", "value": { - "has_valid_signature": { "type": "bool", "value": true }, - "path": { "type": "unicode", "value": "test" }, - "size": { "type": "ByteSize", "value": 17 }, - "timestamp": { "type": "RDFDatetime", "value": 43000000 }, - "type": { "type": "EnumNamedValue", "value": "PYTHON_HACK" } - } - } - ] - }, - "test_class": "ApiListGrrBinariesHandlerRegressionTest_http_v1", - "type_stripped_response": { - "items": [ - { + "has_valid_signature": { + "type": "bool", + "value": true + }, + "path": { + "type": "unicode", + "value": "test" + }, + "size": { + "type": "ByteSize", + "value": 17 + }, + "timestamp": { + "type": "RDFDatetime", + "value": 43000000 + }, + "type": { + "type": "EnumNamedValue", + "value": "PYTHON_HACK" + } + } + } + ] + }, + "test_class": "ApiListGrrBinariesHandlerRegressionTest_http_v1", + "type_stripped_response": { + "items": [ + { "has_valid_signature": true, "path": "windows/test.exe", "size": 18, @@ -9992,7 +14624,12 @@ { "type": "ApiHuntApproval", "value": { - "approvers": [{ "type": "unicode", "value": "api_test_user" }], + "approvers": [ + { + "type": "unicode", + "value": "api_test_user" + } + ], "email_cc_addresses": [], "email_message_id": { "type": "unicode", @@ -10002,21 +14639,47 @@ "type": "RDFDatetime", "value": 2419243000000 }, - "id": { "type": "unicode", "value": "approval:112233" }, - "is_valid": { "type": "bool", "value": false }, + "id": { + "type": "unicode", + "value": "approval:112233" + }, + "is_valid": { + "type": "bool", + "value": false + }, "is_valid_message": { "type": "unicode", "value": "Need at least 1 additional approver for access." }, - "notified_users": [{ "type": "unicode", "value": "approver" }], - "reason": { "type": "unicode", "value": "Running tests" }, - "requestor": { "type": "unicode", "value": "api_test_user" }, + "notified_users": [ + { + "type": "unicode", + "value": "approver" + } + ], + "reason": { + "type": "unicode", + "value": "Running tests" + }, + "requestor": { + "type": "unicode", + "value": "api_test_user" + }, "subject": { "type": "ApiHunt", "value": { - "all_clients_count": { "type": "long", "value": 0 }, - "client_limit": { "type": "long", "value": 100 }, - "client_rate": { "type": "float", "value": 0.0 }, + "all_clients_count": { + "type": "long", + "value": 0 + }, + "client_limit": { + "type": "long", + "value": 100 + }, + "client_rate": { + "type": "float", + "value": 0.0 + }, "client_rule_set": { "type": "ForemanClientRuleSet", "value": { @@ -10046,15 +14709,42 @@ ] } }, - "clients_with_results_count": { "type": "long", "value": 0 }, - "completed_clients_count": { "type": "long", "value": 0 }, - "crash_limit": { "type": "long", "value": 100 }, - "crashed_clients_count": { "type": "long", "value": 0 }, - "created": { "type": "RDFDatetime", "value": 42000000 }, - "creator": { "type": "unicode", "value": "api_test_user" }, - "description": { "type": "unicode", "value": "foo" }, - "duration": { "type": "DurationSeconds", "value": 1209600 }, - "failed_clients_count": { "type": "long", "value": 0 }, + "clients_with_results_count": { + "type": "long", + "value": 0 + }, + "completed_clients_count": { + "type": "long", + "value": 0 + }, + "crash_limit": { + "type": "long", + "value": 100 + }, + "crashed_clients_count": { + "type": "long", + "value": 0 + }, + "created": { + "type": "RDFDatetime", + "value": 42000000 + }, + "creator": { + "type": "unicode", + "value": "api_test_user" + }, + "description": { + "type": "unicode", + "value": "foo" + }, + "duration": { + "type": "DurationSeconds", + "value": 1209600 + }, + "failed_clients_count": { + "type": "long", + "value": 0 + }, "flow_args": { "type": "GetFileArgs", "value": { @@ -10073,8 +14763,14 @@ } } }, - "flow_name": { "type": "unicode", "value": "GetFile" }, - "hunt_id": { "type": "ApiHuntId", "value": "H:123456" }, + "flow_name": { + "type": "unicode", + "value": "GetFile" + }, + "hunt_id": { + "type": "ApiHuntId", + "value": "H:123456" + }, "hunt_runner_args": { "type": "HuntRunnerArgs", "value": { @@ -10090,7 +14786,10 @@ "type": "long", "value": 1000 }, - "client_rate": { "type": "float", "value": 0.0 }, + "client_rate": { + "type": "float", + "value": 0.0 + }, "client_rule_set": { "type": "ForemanClientRuleSet", "value": { @@ -10120,8 +14819,14 @@ ] } }, - "crash_limit": { "type": "long", "value": 100 }, - "description": { "type": "unicode", "value": "foo" }, + "crash_limit": { + "type": "long", + "value": 100 + }, + "description": { + "type": "unicode", + "value": "foo" + }, "expiry_time": { "type": "DurationSeconds", "value": 1209600 @@ -10140,14 +14845,42 @@ "type": "EnumNamedValue", "value": "STANDARD" }, - "is_robot": { "type": "bool", "value": false }, - "name": { "type": "unicode", "value": "GenericHunt" }, - "remaining_clients_count": { "type": "long", "value": 0 }, - "results_count": { "type": "long", "value": 0 }, - "state": { "type": "EnumNamedValue", "value": "PAUSED" }, - "state_comment": { "type": "unicode", "value": "" }, - "total_cpu_usage": { "type": "long", "value": 0 }, - "total_net_usage": { "type": "long", "value": 0 }, + "is_robot": { + "type": "bool", + "value": false + }, + "name": { + "type": "unicode", + "value": "GenericHunt" + }, + "remaining_clients_count": { + "type": "long", + "value": 0 + }, + "results_count": { + "type": "long", + "value": 0 + }, + "state": { + "type": "EnumNamedValue", + "value": "PAUSED" + }, + "state_comment": { + "type": "unicode", + "value": "" + }, + "state_reason": { + "type": "EnumNamedValue", + "value": "UNKNOWN" + }, + "total_cpu_usage": { + "type": "long", + "value": 0 + }, + "total_net_usage": { + "type": "long", + "value": 0 + }, "urn": { "type": "SessionID", "value": "aff4:/hunts/H:123456" @@ -10162,14 +14895,18 @@ "type_stripped_response": { "items": [ { - "approvers": ["api_test_user"], + "approvers": [ + "api_test_user" + ], "email_cc_addresses": [], "email_message_id": "", "expiration_time_us": 2419243000000, "id": "approval:112233", "is_valid": false, "is_valid_message": "Need at least 1 additional approver for access.", - "notified_users": ["approver"], + "notified_users": [ + "approver" + ], "reason": "Running tests", "requestor": "api_test_user", "subject": { @@ -10197,7 +14934,10 @@ "duration": 1209600, "failed_clients_count": 0, "flow_args": { - "pathspec": { "path": "/tmp/evil.txt", "pathtype": "OS" } + "pathspec": { + "path": "/tmp/evil.txt", + "pathtype": "OS" + } }, "flow_name": "GetFile", "hunt_id": "H:123456", @@ -10230,6 +14970,7 @@ "results_count": 0, "state": "PAUSED", "state_comment": "", + "state_reason": "UNKNOWN", "total_cpu_usage": 0, "total_net_usage": 0, "urn": "aff4:/hunts/H:123456" @@ -10253,7 +14994,10 @@ "type": "ApiClientId", "value": "C.1000000000000000" }, - "flow_id": { "type": "ApiFlowId", "value": "H:123456" } + "flow_id": { + "type": "ApiFlowId", + "value": "H:123456" + } } }, { @@ -10263,7 +15007,10 @@ "type": "ApiClientId", "value": "C.1000000000000001" }, - "flow_id": { "type": "ApiFlowId", "value": "H:123456" } + "flow_id": { + "type": "ApiFlowId", + "value": "H:123456" + } } }, { @@ -10273,7 +15020,10 @@ "type": "ApiClientId", "value": "C.1000000000000002" }, - "flow_id": { "type": "ApiFlowId", "value": "H:123456" } + "flow_id": { + "type": "ApiFlowId", + "value": "H:123456" + } } }, { @@ -10283,7 +15033,10 @@ "type": "ApiClientId", "value": "C.1000000000000003" }, - "flow_id": { "type": "ApiFlowId", "value": "H:123456" } + "flow_id": { + "type": "ApiFlowId", + "value": "H:123456" + } } }, { @@ -10293,7 +15046,10 @@ "type": "ApiClientId", "value": "C.1000000000000004" }, - "flow_id": { "type": "ApiFlowId", "value": "H:123456" } + "flow_id": { + "type": "ApiFlowId", + "value": "H:123456" + } } } ], @@ -10302,11 +15058,26 @@ "test_class": "ApiListHuntClientsHandlerRegressionTest_http_v1", "type_stripped_response": { "items": [ - { "client_id": "C.1000000000000000", "flow_id": "H:123456" }, - { "client_id": "C.1000000000000001", "flow_id": "H:123456" }, - { "client_id": "C.1000000000000002", "flow_id": "H:123456" }, - { "client_id": "C.1000000000000003", "flow_id": "H:123456" }, - { "client_id": "C.1000000000000004", "flow_id": "H:123456" } + { + "client_id": "C.1000000000000000", + "flow_id": "H:123456" + }, + { + "client_id": "C.1000000000000001", + "flow_id": "H:123456" + }, + { + "client_id": "C.1000000000000002", + "flow_id": "H:123456" + }, + { + "client_id": "C.1000000000000003", + "flow_id": "H:123456" + }, + { + "client_id": "C.1000000000000004", + "flow_id": "H:123456" + } ], "total_count": 5 }, @@ -10324,7 +15095,10 @@ "type": "ApiClientId", "value": "C.1000000000000000" }, - "flow_id": { "type": "ApiFlowId", "value": "H:123456" } + "flow_id": { + "type": "ApiFlowId", + "value": "H:123456" + } } }, { @@ -10334,7 +15108,10 @@ "type": "ApiClientId", "value": "C.1000000000000001" }, - "flow_id": { "type": "ApiFlowId", "value": "H:123456" } + "flow_id": { + "type": "ApiFlowId", + "value": "H:123456" + } } }, { @@ -10344,7 +15121,10 @@ "type": "ApiClientId", "value": "C.1000000000000002" }, - "flow_id": { "type": "ApiFlowId", "value": "H:123456" } + "flow_id": { + "type": "ApiFlowId", + "value": "H:123456" + } } }, { @@ -10354,7 +15134,10 @@ "type": "ApiClientId", "value": "C.1000000000000003" }, - "flow_id": { "type": "ApiFlowId", "value": "H:123456" } + "flow_id": { + "type": "ApiFlowId", + "value": "H:123456" + } } } ], @@ -10363,10 +15146,22 @@ "test_class": "ApiListHuntClientsHandlerRegressionTest_http_v1", "type_stripped_response": { "items": [ - { "client_id": "C.1000000000000000", "flow_id": "H:123456" }, - { "client_id": "C.1000000000000001", "flow_id": "H:123456" }, - { "client_id": "C.1000000000000002", "flow_id": "H:123456" }, - { "client_id": "C.1000000000000003", "flow_id": "H:123456" } + { + "client_id": "C.1000000000000000", + "flow_id": "H:123456" + }, + { + "client_id": "C.1000000000000001", + "flow_id": "H:123456" + }, + { + "client_id": "C.1000000000000002", + "flow_id": "H:123456" + }, + { + "client_id": "C.1000000000000003", + "flow_id": "H:123456" + } ], "total_count": 4 }, @@ -10384,7 +15179,10 @@ "type": "ApiClientId", "value": "C.1000000000000004" }, - "flow_id": { "type": "ApiFlowId", "value": "H:123456" } + "flow_id": { + "type": "ApiFlowId", + "value": "H:123456" + } } } ], @@ -10392,7 +15190,12 @@ }, "test_class": "ApiListHuntClientsHandlerRegressionTest_http_v1", "type_stripped_response": { - "items": [{ "client_id": "C.1000000000000004", "flow_id": "H:123456" }], + "items": [ + { + "client_id": "C.1000000000000004", + "flow_id": "H:123456" + } + ], "total_count": 1 }, "url": "/api/hunts/H:123456/clients/completed" @@ -10414,12 +15217,27 @@ "client_info": { "type": "ClientInformation", "value": { - "build_time": { "type": "unicode", "value": "1980-01-01" }, - "client_name": { "type": "unicode", "value": "GRR Monitor" }, - "client_version": { "type": "long", "value": 1234 }, + "build_time": { + "type": "unicode", + "value": "1980-01-01" + }, + "client_name": { + "type": "unicode", + "value": "GRR Monitor" + }, + "client_version": { + "type": "long", + "value": 1234 + }, "labels": [ - { "type": "unicode", "value": "label1" }, - { "type": "unicode", "value": "label2" } + { + "type": "unicode", + "value": "label1" + }, + { + "type": "unicode", + "value": "label2" + } ] } }, @@ -10427,12 +15245,18 @@ "type": "unicode", "value": "Client killed during transaction" }, - "crash_type": { "type": "unicode", "value": "Client Crash" }, + "crash_type": { + "type": "unicode", + "value": "Client Crash" + }, "session_id": { "type": "SessionID", "value": "aff4:/hunts/H:123456/C.1000000000000000/H:11223344" }, - "timestamp": { "type": "RDFDatetime", "value": 45000000 } + "timestamp": { + "type": "RDFDatetime", + "value": 45000000 + } } } ], @@ -10447,7 +15271,10 @@ "build_time": "1980-01-01", "client_name": "GRR Monitor", "client_version": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "crash_message": "Client killed during transaction", "crash_type": "Client Crash", @@ -10474,12 +15301,27 @@ "client_info": { "type": "ClientInformation", "value": { - "build_time": { "type": "unicode", "value": "1980-01-01" }, - "client_name": { "type": "unicode", "value": "GRR Monitor" }, - "client_version": { "type": "long", "value": 1234 }, + "build_time": { + "type": "unicode", + "value": "1980-01-01" + }, + "client_name": { + "type": "unicode", + "value": "GRR Monitor" + }, + "client_version": { + "type": "long", + "value": 1234 + }, "labels": [ - { "type": "unicode", "value": "label1" }, - { "type": "unicode", "value": "label2" } + { + "type": "unicode", + "value": "label1" + }, + { + "type": "unicode", + "value": "label2" + } ] } }, @@ -10487,12 +15329,18 @@ "type": "unicode", "value": "Client killed during transaction" }, - "crash_type": { "type": "unicode", "value": "Client Crash" }, + "crash_type": { + "type": "unicode", + "value": "Client Crash" + }, "session_id": { "type": "SessionID", "value": "aff4:/hunts/H:123456/C.1000000000000000/H:11223344" }, - "timestamp": { "type": "RDFDatetime", "value": 45000000 } + "timestamp": { + "type": "RDFDatetime", + "value": 45000000 + } } } ], @@ -10507,7 +15355,10 @@ "build_time": "1980-01-01", "client_name": "GRR Monitor", "client_version": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "crash_message": "Client killed during transaction", "crash_type": "Client Crash", @@ -10522,7 +15373,10 @@ { "api_method": "ListHuntCrashes", "method": "GET", - "response": { "items": [], "total_count": 1 }, + "response": { + "items": [], + "total_count": 1 + }, "test_class": "ApiListHuntCrashesHandlerRegressionTest_http_v1", "url": "/api/hunts/H:123456/crashes?count=1&offset=1" } @@ -10540,20 +15394,35 @@ "type": "ApiClientId", "value": "C.1000000000000000" }, - "log_message": { "type": "unicode", "value": "Error foo." }, - "timestamp": { "type": "RDFDatetime", "value": 52000000 } + "log_message": { + "type": "unicode", + "value": "Error foo." + }, + "timestamp": { + "type": "RDFDatetime", + "value": 52000000 + } } }, { "type": "ApiHuntError", "value": { - "backtrace": { "type": "unicode", "value": "" }, + "backtrace": { + "type": "unicode", + "value": "" + }, "client_id": { "type": "ApiClientId", "value": "C.1000000000000001" }, - "log_message": { "type": "unicode", "value": "Error bar." }, - "timestamp": { "type": "RDFDatetime", "value": 55000000 } + "log_message": { + "type": "unicode", + "value": "Error bar." + }, + "timestamp": { + "type": "RDFDatetime", + "value": 55000000 + } } } ], @@ -10590,8 +15459,14 @@ "type": "ApiClientId", "value": "C.1000000000000000" }, - "log_message": { "type": "unicode", "value": "Error foo." }, - "timestamp": { "type": "RDFDatetime", "value": 52000000 } + "log_message": { + "type": "unicode", + "value": "Error foo." + }, + "timestamp": { + "type": "RDFDatetime", + "value": 52000000 + } } } ], @@ -10618,13 +15493,22 @@ { "type": "ApiHuntError", "value": { - "backtrace": { "type": "unicode", "value": "" }, + "backtrace": { + "type": "unicode", + "value": "" + }, "client_id": { "type": "ApiClientId", "value": "C.1000000000000001" }, - "log_message": { "type": "unicode", "value": "Error bar." }, - "timestamp": { "type": "RDFDatetime", "value": 55000000 } + "log_message": { + "type": "unicode", + "value": "Error bar." + }, + "timestamp": { + "type": "RDFDatetime", + "value": 55000000 + } } } ], @@ -10658,13 +15542,22 @@ "type": "ApiClientId", "value": "C.1000000000000000" }, - "flow_id": { "type": "ApiFlowId", "value": "H:123456" }, - "flow_name": { "type": "unicode", "value": "GenericHunt" }, + "flow_id": { + "type": "ApiFlowId", + "value": "H:123456" + }, + "flow_name": { + "type": "unicode", + "value": "GenericHunt" + }, "log_message": { "type": "unicode", "value": "Sample message: foo" }, - "timestamp": { "type": "RDFDatetime", "value": 52000000 } + "timestamp": { + "type": "RDFDatetime", + "value": 52000000 + } } }, { @@ -10674,13 +15567,22 @@ "type": "ApiClientId", "value": "C.1000000000000000" }, - "flow_id": { "type": "ApiFlowId", "value": "H:123456" }, - "flow_name": { "type": "unicode", "value": "GenericHunt" }, + "flow_id": { + "type": "ApiFlowId", + "value": "H:123456" + }, + "flow_name": { + "type": "unicode", + "value": "GenericHunt" + }, "log_message": { "type": "unicode", "value": "Sample message: bar" }, - "timestamp": { "type": "RDFDatetime", "value": 55000000 } + "timestamp": { + "type": "RDFDatetime", + "value": 55000000 + } } } ], @@ -10720,13 +15622,22 @@ "type": "ApiClientId", "value": "C.1000000000000000" }, - "flow_id": { "type": "ApiFlowId", "value": "H:123456" }, - "flow_name": { "type": "unicode", "value": "GenericHunt" }, + "flow_id": { + "type": "ApiFlowId", + "value": "H:123456" + }, + "flow_name": { + "type": "unicode", + "value": "GenericHunt" + }, "log_message": { "type": "unicode", "value": "Sample message: foo" }, - "timestamp": { "type": "RDFDatetime", "value": 52000000 } + "timestamp": { + "type": "RDFDatetime", + "value": 52000000 + } } } ], @@ -10759,13 +15670,22 @@ "type": "ApiClientId", "value": "C.1000000000000000" }, - "flow_id": { "type": "ApiFlowId", "value": "H:123456" }, - "flow_name": { "type": "unicode", "value": "GenericHunt" }, + "flow_id": { + "type": "ApiFlowId", + "value": "H:123456" + }, + "flow_name": { + "type": "unicode", + "value": "GenericHunt" + }, "log_message": { "type": "unicode", "value": "Sample message: bar" }, - "timestamp": { "type": "RDFDatetime", "value": 55000000 } + "timestamp": { + "type": "RDFDatetime", + "value": 55000000 + } } } ], @@ -10796,9 +15716,18 @@ { "type": "OutputPluginBatchProcessingStatus", "value": { - "batch_index": { "type": "long", "value": 0 }, - "batch_size": { "type": "long", "value": 0 }, - "status": { "type": "EnumNamedValue", "value": "ERROR" }, + "batch_index": { + "type": "long", + "value": 0 + }, + "batch_size": { + "type": "long", + "value": 0 + }, + "status": { + "type": "EnumNamedValue", + "value": "ERROR" + }, "summary": { "type": "unicode", "value": "Error while processing 1 replies: Oh no!" @@ -10808,9 +15737,18 @@ { "type": "OutputPluginBatchProcessingStatus", "value": { - "batch_index": { "type": "long", "value": 0 }, - "batch_size": { "type": "long", "value": 0 }, - "status": { "type": "EnumNamedValue", "value": "ERROR" }, + "batch_index": { + "type": "long", + "value": 0 + }, + "batch_size": { + "type": "long", + "value": 0 + }, + "status": { + "type": "EnumNamedValue", + "value": "ERROR" + }, "summary": { "type": "unicode", "value": "Error while processing 1 replies: Oh no!" @@ -10850,19 +15788,43 @@ { "type": "OutputPluginBatchProcessingStatus", "value": { - "batch_index": { "type": "long", "value": 0 }, - "batch_size": { "type": "long", "value": 0 }, - "status": { "type": "EnumNamedValue", "value": "SUCCESS" }, - "summary": { "type": "unicode", "value": "Processed 1 replies." } + "batch_index": { + "type": "long", + "value": 0 + }, + "batch_size": { + "type": "long", + "value": 0 + }, + "status": { + "type": "EnumNamedValue", + "value": "SUCCESS" + }, + "summary": { + "type": "unicode", + "value": "Processed 1 replies." + } } }, { "type": "OutputPluginBatchProcessingStatus", "value": { - "batch_index": { "type": "long", "value": 0 }, - "batch_size": { "type": "long", "value": 0 }, - "status": { "type": "EnumNamedValue", "value": "SUCCESS" }, - "summary": { "type": "unicode", "value": "Processed 1 replies." } + "batch_index": { + "type": "long", + "value": 0 + }, + "batch_size": { + "type": "long", + "value": 0 + }, + "status": { + "type": "EnumNamedValue", + "value": "SUCCESS" + }, + "summary": { + "type": "unicode", + "value": "Processed 1 replies." + } } } ], @@ -10908,7 +15870,10 @@ "args": { "type": "ListProcessesArgs", "value": { - "fetch_binaries": { "type": "bool", "value": true }, + "fetch_binaries": { + "type": "bool", + "value": true + }, "filename_regex": { "type": "RegularExpression", "value": "blah!" @@ -10927,7 +15892,10 @@ "args": { "type": "ListProcessesArgs", "value": { - "fetch_binaries": { "type": "bool", "value": true }, + "fetch_binaries": { + "type": "bool", + "value": true + }, "filename_regex": { "type": "RegularExpression", "value": "blah!" @@ -10947,11 +15915,17 @@ { "id": "DummyHuntTestOutputPlugin_0", "plugin_descriptor": { - "args": { "fetch_binaries": true, "filename_regex": "blah!" }, + "args": { + "fetch_binaries": true, + "filename_regex": "blah!" + }, "plugin_name": "DummyHuntTestOutputPlugin" }, "state": { - "args": { "fetch_binaries": true, "filename_regex": "blah!" } + "args": { + "fetch_binaries": true, + "filename_regex": "blah!" + } } } ], @@ -10973,9 +15947,18 @@ "type": "ApiClientId", "value": "C.1000000000000000" }, - "payload": { "type": "RDFString", "value": "blah1" }, - "payload_type": { "type": "unicode", "value": "RDFString" }, - "timestamp": { "type": "RDFDatetime", "value": 2000000 } + "payload": { + "type": "RDFString", + "value": "blah1" + }, + "payload_type": { + "type": "unicode", + "value": "RDFString" + }, + "timestamp": { + "type": "RDFDatetime", + "value": 2000000 + } } }, { @@ -10985,12 +15968,21 @@ "type": "ApiClientId", "value": "C.1000000000000000" }, - "payload": { "type": "RDFString", "value": "blah2-foo" }, - "payload_type": { "type": "unicode", "value": "RDFString" }, - "timestamp": { "type": "RDFDatetime", "value": 43000000 } - } - } - ], + "payload": { + "type": "RDFString", + "value": "blah2-foo" + }, + "payload_type": { + "type": "unicode", + "value": "RDFString" + }, + "timestamp": { + "type": "RDFDatetime", + "value": 43000000 + } + } + } + ], "total_count": 2 }, "test_class": "ApiListHuntResultsRegressionTest_http_v1", @@ -11025,9 +16017,18 @@ "type": "ApiClientId", "value": "C.1000000000000000" }, - "payload": { "type": "RDFString", "value": "blah1" }, - "payload_type": { "type": "unicode", "value": "RDFString" }, - "timestamp": { "type": "RDFDatetime", "value": 2000000 } + "payload": { + "type": "RDFString", + "value": "blah1" + }, + "payload_type": { + "type": "unicode", + "value": "RDFString" + }, + "timestamp": { + "type": "RDFDatetime", + "value": 2000000 + } } } ], @@ -11059,9 +16060,18 @@ "type": "ApiClientId", "value": "C.1000000000000000" }, - "payload": { "type": "RDFString", "value": "blah2-foo" }, - "payload_type": { "type": "unicode", "value": "RDFString" }, - "timestamp": { "type": "RDFDatetime", "value": 43000000 } + "payload": { + "type": "RDFString", + "value": "blah2-foo" + }, + "payload_type": { + "type": "unicode", + "value": "RDFString" + }, + "timestamp": { + "type": "RDFDatetime", + "value": 43000000 + } } } ], @@ -11093,9 +16103,18 @@ "type": "ApiClientId", "value": "C.1000000000000000" }, - "payload": { "type": "RDFString", "value": "blah2-foo" }, - "payload_type": { "type": "unicode", "value": "RDFString" }, - "timestamp": { "type": "RDFDatetime", "value": 43000000 } + "payload": { + "type": "RDFString", + "value": "blah2-foo" + }, + "payload_type": { + "type": "unicode", + "value": "RDFString" + }, + "timestamp": { + "type": "RDFDatetime", + "value": 43000000 + } } } ], @@ -11125,31 +16144,91 @@ { "type": "ApiHunt", "value": { - "client_limit": { "type": "long", "value": 100 }, - "client_rate": { "type": "float", "value": 0.0 }, - "created": { "type": "RDFDatetime", "value": 2000000000 }, - "creator": { "type": "unicode", "value": "api_test_user" }, - "description": { "type": "unicode", "value": "hunt_1" }, - "duration": { "type": "DurationSeconds", "value": 1209600 }, - "hunt_id": { "type": "ApiHuntId", "value": "H:000001" }, - "is_robot": { "type": "bool", "value": false }, - "state": { "type": "EnumNamedValue", "value": "STOPPED" }, - "urn": { "type": "SessionID", "value": "aff4:/hunts/H:000001" } + "client_limit": { + "type": "long", + "value": 100 + }, + "client_rate": { + "type": "float", + "value": 0.0 + }, + "created": { + "type": "RDFDatetime", + "value": 2000000000 + }, + "creator": { + "type": "unicode", + "value": "api_test_user" + }, + "description": { + "type": "unicode", + "value": "hunt_1" + }, + "duration": { + "type": "DurationSeconds", + "value": 1209600 + }, + "hunt_id": { + "type": "ApiHuntId", + "value": "H:000001" + }, + "is_robot": { + "type": "bool", + "value": false + }, + "state": { + "type": "EnumNamedValue", + "value": "STOPPED" + }, + "urn": { + "type": "SessionID", + "value": "aff4:/hunts/H:000001" + } } }, { "type": "ApiHunt", "value": { - "client_limit": { "type": "long", "value": 100 }, - "client_rate": { "type": "float", "value": 0.0 }, - "created": { "type": "RDFDatetime", "value": 1000000000 }, - "creator": { "type": "unicode", "value": "GRRWorker" }, - "description": { "type": "unicode", "value": "hunt_0" }, - "duration": { "type": "DurationSeconds", "value": 1209600 }, - "hunt_id": { "type": "ApiHuntId", "value": "H:000000" }, - "is_robot": { "type": "bool", "value": true }, - "state": { "type": "EnumNamedValue", "value": "PAUSED" }, - "urn": { "type": "SessionID", "value": "aff4:/hunts/H:000000" } + "client_limit": { + "type": "long", + "value": 100 + }, + "client_rate": { + "type": "float", + "value": 0.0 + }, + "created": { + "type": "RDFDatetime", + "value": 1000000000 + }, + "creator": { + "type": "unicode", + "value": "GRRWorker" + }, + "description": { + "type": "unicode", + "value": "hunt_0" + }, + "duration": { + "type": "DurationSeconds", + "value": 1209600 + }, + "hunt_id": { + "type": "ApiHuntId", + "value": "H:000000" + }, + "is_robot": { + "type": "bool", + "value": true + }, + "state": { + "type": "EnumNamedValue", + "value": "PAUSED" + }, + "urn": { + "type": "SessionID", + "value": "aff4:/hunts/H:000000" + } } } ] @@ -11193,16 +16272,46 @@ { "type": "ApiHunt", "value": { - "client_limit": { "type": "long", "value": 100 }, - "client_rate": { "type": "float", "value": 0.0 }, - "created": { "type": "RDFDatetime", "value": 2000000000 }, - "creator": { "type": "unicode", "value": "api_test_user" }, - "description": { "type": "unicode", "value": "hunt_1" }, - "duration": { "type": "DurationSeconds", "value": 1209600 }, - "hunt_id": { "type": "ApiHuntId", "value": "H:000001" }, - "is_robot": { "type": "bool", "value": false }, - "state": { "type": "EnumNamedValue", "value": "STOPPED" }, - "urn": { "type": "SessionID", "value": "aff4:/hunts/H:000001" } + "client_limit": { + "type": "long", + "value": 100 + }, + "client_rate": { + "type": "float", + "value": 0.0 + }, + "created": { + "type": "RDFDatetime", + "value": 2000000000 + }, + "creator": { + "type": "unicode", + "value": "api_test_user" + }, + "description": { + "type": "unicode", + "value": "hunt_1" + }, + "duration": { + "type": "DurationSeconds", + "value": 1209600 + }, + "hunt_id": { + "type": "ApiHuntId", + "value": "H:000001" + }, + "is_robot": { + "type": "bool", + "value": false + }, + "state": { + "type": "EnumNamedValue", + "value": "STOPPED" + }, + "urn": { + "type": "SessionID", + "value": "aff4:/hunts/H:000001" + } } } ] @@ -11234,16 +16343,46 @@ { "type": "ApiHunt", "value": { - "client_limit": { "type": "long", "value": 100 }, - "client_rate": { "type": "float", "value": 0.0 }, - "created": { "type": "RDFDatetime", "value": 1000000000 }, - "creator": { "type": "unicode", "value": "GRRWorker" }, - "description": { "type": "unicode", "value": "hunt_0" }, - "duration": { "type": "DurationSeconds", "value": 1209600 }, - "hunt_id": { "type": "ApiHuntId", "value": "H:000000" }, - "is_robot": { "type": "bool", "value": true }, - "state": { "type": "EnumNamedValue", "value": "PAUSED" }, - "urn": { "type": "SessionID", "value": "aff4:/hunts/H:000000" } + "client_limit": { + "type": "long", + "value": 100 + }, + "client_rate": { + "type": "float", + "value": 0.0 + }, + "created": { + "type": "RDFDatetime", + "value": 1000000000 + }, + "creator": { + "type": "unicode", + "value": "GRRWorker" + }, + "description": { + "type": "unicode", + "value": "hunt_0" + }, + "duration": { + "type": "DurationSeconds", + "value": 1209600 + }, + "hunt_id": { + "type": "ApiHuntId", + "value": "H:000000" + }, + "is_robot": { + "type": "bool", + "value": true + }, + "state": { + "type": "EnumNamedValue", + "value": "PAUSED" + }, + "urn": { + "type": "SessionID", + "value": "aff4:/hunts/H:000000" + } } } ] @@ -11275,16 +16414,46 @@ { "type": "ApiHunt", "value": { - "client_limit": { "type": "long", "value": 100 }, - "client_rate": { "type": "float", "value": 0.0 }, - "created": { "type": "RDFDatetime", "value": 2000000000 }, - "creator": { "type": "unicode", "value": "api_test_user" }, - "description": { "type": "unicode", "value": "hunt_1" }, - "duration": { "type": "DurationSeconds", "value": 1209600 }, - "hunt_id": { "type": "ApiHuntId", "value": "H:000001" }, - "is_robot": { "type": "bool", "value": false }, - "state": { "type": "EnumNamedValue", "value": "STOPPED" }, - "urn": { "type": "SessionID", "value": "aff4:/hunts/H:000001" } + "client_limit": { + "type": "long", + "value": 100 + }, + "client_rate": { + "type": "float", + "value": 0.0 + }, + "created": { + "type": "RDFDatetime", + "value": 2000000000 + }, + "creator": { + "type": "unicode", + "value": "api_test_user" + }, + "description": { + "type": "unicode", + "value": "hunt_1" + }, + "duration": { + "type": "DurationSeconds", + "value": 1209600 + }, + "hunt_id": { + "type": "ApiHuntId", + "value": "H:000001" + }, + "is_robot": { + "type": "bool", + "value": false + }, + "state": { + "type": "EnumNamedValue", + "value": "STOPPED" + }, + "urn": { + "type": "SessionID", + "value": "aff4:/hunts/H:000001" + } } } ] @@ -11316,16 +16485,46 @@ { "type": "ApiHunt", "value": { - "client_limit": { "type": "long", "value": 100 }, - "client_rate": { "type": "float", "value": 0.0 }, - "created": { "type": "RDFDatetime", "value": 1000000000 }, - "creator": { "type": "unicode", "value": "GRRWorker" }, - "description": { "type": "unicode", "value": "hunt_0" }, - "duration": { "type": "DurationSeconds", "value": 1209600 }, - "hunt_id": { "type": "ApiHuntId", "value": "H:000000" }, - "is_robot": { "type": "bool", "value": true }, - "state": { "type": "EnumNamedValue", "value": "PAUSED" }, - "urn": { "type": "SessionID", "value": "aff4:/hunts/H:000000" } + "client_limit": { + "type": "long", + "value": 100 + }, + "client_rate": { + "type": "float", + "value": 0.0 + }, + "created": { + "type": "RDFDatetime", + "value": 1000000000 + }, + "creator": { + "type": "unicode", + "value": "GRRWorker" + }, + "description": { + "type": "unicode", + "value": "hunt_0" + }, + "duration": { + "type": "DurationSeconds", + "value": 1209600 + }, + "hunt_id": { + "type": "ApiHuntId", + "value": "H:000000" + }, + "is_robot": { + "type": "bool", + "value": true + }, + "state": { + "type": "EnumNamedValue", + "value": "PAUSED" + }, + "urn": { + "type": "SessionID", + "value": "aff4:/hunts/H:000000" + } } } ] @@ -11357,31 +16556,91 @@ { "type": "ApiHunt", "value": { - "client_limit": { "type": "long", "value": 100 }, - "client_rate": { "type": "float", "value": 0.0 }, - "created": { "type": "RDFDatetime", "value": 2000000000 }, - "creator": { "type": "unicode", "value": "api_test_user" }, - "description": { "type": "unicode", "value": "hunt_1" }, - "duration": { "type": "DurationSeconds", "value": 1209600 }, - "hunt_id": { "type": "ApiHuntId", "value": "H:000001" }, - "is_robot": { "type": "bool", "value": false }, - "state": { "type": "EnumNamedValue", "value": "STOPPED" }, - "urn": { "type": "SessionID", "value": "aff4:/hunts/H:000001" } + "client_limit": { + "type": "long", + "value": 100 + }, + "client_rate": { + "type": "float", + "value": 0.0 + }, + "created": { + "type": "RDFDatetime", + "value": 2000000000 + }, + "creator": { + "type": "unicode", + "value": "api_test_user" + }, + "description": { + "type": "unicode", + "value": "hunt_1" + }, + "duration": { + "type": "DurationSeconds", + "value": 1209600 + }, + "hunt_id": { + "type": "ApiHuntId", + "value": "H:000001" + }, + "is_robot": { + "type": "bool", + "value": false + }, + "state": { + "type": "EnumNamedValue", + "value": "STOPPED" + }, + "urn": { + "type": "SessionID", + "value": "aff4:/hunts/H:000001" + } } }, { "type": "ApiHunt", "value": { - "client_limit": { "type": "long", "value": 100 }, - "client_rate": { "type": "float", "value": 0.0 }, - "created": { "type": "RDFDatetime", "value": 1000000000 }, - "creator": { "type": "unicode", "value": "GRRWorker" }, - "description": { "type": "unicode", "value": "hunt_0" }, - "duration": { "type": "DurationSeconds", "value": 1209600 }, - "hunt_id": { "type": "ApiHuntId", "value": "H:000000" }, - "is_robot": { "type": "bool", "value": true }, - "state": { "type": "EnumNamedValue", "value": "PAUSED" }, - "urn": { "type": "SessionID", "value": "aff4:/hunts/H:000000" } + "client_limit": { + "type": "long", + "value": 100 + }, + "client_rate": { + "type": "float", + "value": 0.0 + }, + "created": { + "type": "RDFDatetime", + "value": 1000000000 + }, + "creator": { + "type": "unicode", + "value": "GRRWorker" + }, + "description": { + "type": "unicode", + "value": "hunt_0" + }, + "duration": { + "type": "DurationSeconds", + "value": 1209600 + }, + "hunt_id": { + "type": "ApiHuntId", + "value": "H:000000" + }, + "is_robot": { + "type": "bool", + "value": true + }, + "state": { + "type": "EnumNamedValue", + "value": "PAUSED" + }, + "urn": { + "type": "SessionID", + "value": "aff4:/hunts/H:000000" + } } } ] @@ -11425,31 +16684,91 @@ { "type": "ApiHunt", "value": { - "client_limit": { "type": "long", "value": 100 }, - "client_rate": { "type": "float", "value": 0.0 }, - "created": { "type": "RDFDatetime", "value": 2000000000 }, - "creator": { "type": "unicode", "value": "api_test_user" }, - "description": { "type": "unicode", "value": "hunt_1" }, - "duration": { "type": "DurationSeconds", "value": 1209600 }, - "hunt_id": { "type": "ApiHuntId", "value": "H:000001" }, - "is_robot": { "type": "bool", "value": false }, - "state": { "type": "EnumNamedValue", "value": "STOPPED" }, - "urn": { "type": "SessionID", "value": "aff4:/hunts/H:000001" } + "client_limit": { + "type": "long", + "value": 100 + }, + "client_rate": { + "type": "float", + "value": 0.0 + }, + "created": { + "type": "RDFDatetime", + "value": 2000000000 + }, + "creator": { + "type": "unicode", + "value": "api_test_user" + }, + "description": { + "type": "unicode", + "value": "hunt_1" + }, + "duration": { + "type": "DurationSeconds", + "value": 1209600 + }, + "hunt_id": { + "type": "ApiHuntId", + "value": "H:000001" + }, + "is_robot": { + "type": "bool", + "value": false + }, + "state": { + "type": "EnumNamedValue", + "value": "STOPPED" + }, + "urn": { + "type": "SessionID", + "value": "aff4:/hunts/H:000001" + } } }, { "type": "ApiHunt", "value": { - "client_limit": { "type": "long", "value": 100 }, - "client_rate": { "type": "float", "value": 0.0 }, - "created": { "type": "RDFDatetime", "value": 1000000000 }, - "creator": { "type": "unicode", "value": "GRRWorker" }, - "description": { "type": "unicode", "value": "hunt_0" }, - "duration": { "type": "DurationSeconds", "value": 1209600 }, - "hunt_id": { "type": "ApiHuntId", "value": "H:000000" }, - "is_robot": { "type": "bool", "value": true }, - "state": { "type": "EnumNamedValue", "value": "PAUSED" }, - "urn": { "type": "SessionID", "value": "aff4:/hunts/H:000000" } + "client_limit": { + "type": "long", + "value": 100 + }, + "client_rate": { + "type": "float", + "value": 0.0 + }, + "created": { + "type": "RDFDatetime", + "value": 1000000000 + }, + "creator": { + "type": "unicode", + "value": "GRRWorker" + }, + "description": { + "type": "unicode", + "value": "hunt_0" + }, + "duration": { + "type": "DurationSeconds", + "value": 1209600 + }, + "hunt_id": { + "type": "ApiHuntId", + "value": "H:000000" + }, + "is_robot": { + "type": "bool", + "value": true + }, + "state": { + "type": "EnumNamedValue", + "value": "PAUSED" + }, + "urn": { + "type": "SessionID", + "value": "aff4:/hunts/H:000000" + } } } ] @@ -11488,7 +16807,9 @@ { "api_method": "ListHunts", "method": "GET", - "response": { "items": [] }, + "response": { + "items": [] + }, "test_class": "ApiListHuntsHandlerRegressionTest_http_v1", "url": "/api/hunts?count=2&offset=0&with_state=started" } @@ -11499,75 +16820,216 @@ "method": "GET", "response": { "items": [ - { "type": "unicode", "value": "code_page" }, - { "type": "unicode", "value": "current_control_set" }, - { "type": "unicode", "value": "domain" }, - { "type": "unicode", "value": "environ_allusersappdata" }, - { "type": "unicode", "value": "environ_allusersprofile" }, - { "type": "unicode", "value": "environ_commonprogramfiles" }, - { "type": "unicode", "value": "environ_commonprogramfilesx86" }, - { "type": "unicode", "value": "environ_comspec" }, - { "type": "unicode", "value": "environ_driverdata" }, - { "type": "unicode", "value": "environ_path" }, - { "type": "unicode", "value": "environ_profilesdirectory" }, - { "type": "unicode", "value": "environ_programdata" }, - { "type": "unicode", "value": "environ_programfiles" }, - { "type": "unicode", "value": "environ_programfilesx86" }, - { "type": "unicode", "value": "environ_systemdrive" }, - { "type": "unicode", "value": "environ_systemroot" }, - { "type": "unicode", "value": "environ_temp" }, - { "type": "unicode", "value": "environ_windir" }, - { "type": "unicode", "value": "fqdn" }, - { "type": "unicode", "value": "hostname" }, - { "type": "unicode", "value": "os" }, - { "type": "unicode", "value": "os_major_version" }, - { "type": "unicode", "value": "os_minor_version" }, - { "type": "unicode", "value": "os_release" }, - { "type": "unicode", "value": "time_zone" }, - { "type": "unicode", "value": "users.appdata" }, - { "type": "unicode", "value": "users.cookies" }, - { "type": "unicode", "value": "users.desktop" }, - { "type": "unicode", "value": "users.full_name" }, - { "type": "unicode", "value": "users.gid" }, - { "type": "unicode", "value": "users.gids" }, - { "type": "unicode", "value": "users.homedir" }, - { "type": "unicode", "value": "users.internet_cache" }, - { "type": "unicode", "value": "users.last_logon" }, - { "type": "unicode", "value": "users.localappdata" }, - { "type": "unicode", "value": "users.localappdata_low" }, - { "type": "unicode", "value": "users.personal" }, - { "type": "unicode", "value": "users.pw_entry" }, - { "type": "unicode", "value": "users.recent" }, - { "type": "unicode", "value": "users.shell" }, - { "type": "unicode", "value": "users.sid" }, - { "type": "unicode", "value": "users.startup" }, - { "type": "unicode", "value": "users.temp" }, - { "type": "unicode", "value": "users.uid" }, - { "type": "unicode", "value": "users.userdomain" }, - { "type": "unicode", "value": "users.username" }, - { "type": "unicode", "value": "users.userprofile" } - ] - }, - "test_class": "ApiListKbFieldsHandlerTest_http_v1", - "type_stripped_response": { - "items": [ - "code_page", - "current_control_set", - "domain", - "environ_allusersappdata", - "environ_allusersprofile", - "environ_commonprogramfiles", - "environ_commonprogramfilesx86", - "environ_comspec", - "environ_driverdata", - "environ_path", - "environ_profilesdirectory", - "environ_programdata", - "environ_programfiles", - "environ_programfilesx86", - "environ_systemdrive", - "environ_systemroot", - "environ_temp", + { + "type": "unicode", + "value": "code_page" + }, + { + "type": "unicode", + "value": "current_control_set" + }, + { + "type": "unicode", + "value": "domain" + }, + { + "type": "unicode", + "value": "environ_allusersappdata" + }, + { + "type": "unicode", + "value": "environ_allusersprofile" + }, + { + "type": "unicode", + "value": "environ_commonprogramfiles" + }, + { + "type": "unicode", + "value": "environ_commonprogramfilesx86" + }, + { + "type": "unicode", + "value": "environ_comspec" + }, + { + "type": "unicode", + "value": "environ_driverdata" + }, + { + "type": "unicode", + "value": "environ_path" + }, + { + "type": "unicode", + "value": "environ_profilesdirectory" + }, + { + "type": "unicode", + "value": "environ_programdata" + }, + { + "type": "unicode", + "value": "environ_programfiles" + }, + { + "type": "unicode", + "value": "environ_programfilesx86" + }, + { + "type": "unicode", + "value": "environ_systemdrive" + }, + { + "type": "unicode", + "value": "environ_systemroot" + }, + { + "type": "unicode", + "value": "environ_temp" + }, + { + "type": "unicode", + "value": "environ_windir" + }, + { + "type": "unicode", + "value": "fqdn" + }, + { + "type": "unicode", + "value": "hostname" + }, + { + "type": "unicode", + "value": "os" + }, + { + "type": "unicode", + "value": "os_major_version" + }, + { + "type": "unicode", + "value": "os_minor_version" + }, + { + "type": "unicode", + "value": "os_release" + }, + { + "type": "unicode", + "value": "time_zone" + }, + { + "type": "unicode", + "value": "users.appdata" + }, + { + "type": "unicode", + "value": "users.cookies" + }, + { + "type": "unicode", + "value": "users.desktop" + }, + { + "type": "unicode", + "value": "users.full_name" + }, + { + "type": "unicode", + "value": "users.gid" + }, + { + "type": "unicode", + "value": "users.gids" + }, + { + "type": "unicode", + "value": "users.homedir" + }, + { + "type": "unicode", + "value": "users.internet_cache" + }, + { + "type": "unicode", + "value": "users.last_logon" + }, + { + "type": "unicode", + "value": "users.localappdata" + }, + { + "type": "unicode", + "value": "users.localappdata_low" + }, + { + "type": "unicode", + "value": "users.personal" + }, + { + "type": "unicode", + "value": "users.pw_entry" + }, + { + "type": "unicode", + "value": "users.recent" + }, + { + "type": "unicode", + "value": "users.shell" + }, + { + "type": "unicode", + "value": "users.sid" + }, + { + "type": "unicode", + "value": "users.startup" + }, + { + "type": "unicode", + "value": "users.temp" + }, + { + "type": "unicode", + "value": "users.uid" + }, + { + "type": "unicode", + "value": "users.userdomain" + }, + { + "type": "unicode", + "value": "users.username" + }, + { + "type": "unicode", + "value": "users.userprofile" + } + ] + }, + "test_class": "ApiListKbFieldsHandlerTest_http_v1", + "type_stripped_response": { + "items": [ + "code_page", + "current_control_set", + "domain", + "environ_allusersappdata", + "environ_allusersprofile", + "environ_commonprogramfiles", + "environ_commonprogramfilesx86", + "environ_comspec", + "environ_driverdata", + "environ_path", + "environ_profilesdirectory", + "environ_programdata", + "environ_programfiles", + "environ_programfilesx86", + "environ_systemdrive", + "environ_systemroot", + "environ_temp", "environ_windir", "fqdn", "hostname", @@ -11609,58 +17071,214 @@ "method": "GET", "response": { "encodings": [ - { "type": "unicode", "value": "BASE64_CODEC" }, - { "type": "unicode", "value": "BIG5" }, - { "type": "unicode", "value": "BIG5HKSCS" }, - { "type": "unicode", "value": "CP037" }, - { "type": "unicode", "value": "CP1006" }, - { "type": "unicode", "value": "CP1026" }, - { "type": "unicode", "value": "CP1140" }, - { "type": "unicode", "value": "CP1250" }, - { "type": "unicode", "value": "CP1251" }, - { "type": "unicode", "value": "CP1252" }, - { "type": "unicode", "value": "CP1253" }, - { "type": "unicode", "value": "CP1254" }, - { "type": "unicode", "value": "CP1255" }, - { "type": "unicode", "value": "CP1256" }, - { "type": "unicode", "value": "CP1257" }, - { "type": "unicode", "value": "CP1258" }, - { "type": "unicode", "value": "CP424" }, - { "type": "unicode", "value": "CP437" }, - { "type": "unicode", "value": "CP500" }, - { "type": "unicode", "value": "CP737" }, - { "type": "unicode", "value": "CP775" }, - { "type": "unicode", "value": "CP850" }, - { "type": "unicode", "value": "CP852" }, - { "type": "unicode", "value": "CP855" }, - { "type": "unicode", "value": "CP856" }, - { "type": "unicode", "value": "CP857" }, - { "type": "unicode", "value": "CP860" }, - { "type": "unicode", "value": "CP861" }, - { "type": "unicode", "value": "CP862" }, - { "type": "unicode", "value": "CP863" }, - { "type": "unicode", "value": "CP864" }, - { "type": "unicode", "value": "CP865" }, - { "type": "unicode", "value": "CP866" }, - { "type": "unicode", "value": "CP869" }, - { "type": "unicode", "value": "CP874" }, - { "type": "unicode", "value": "CP875" }, - { "type": "unicode", "value": "CP932" }, - { "type": "unicode", "value": "CP949" }, - { "type": "unicode", "value": "CP950" }, - { "type": "unicode", "value": "IDNA" }, - { "type": "unicode", "value": "ROT_13" }, - { "type": "unicode", "value": "UTF_16" }, - { "type": "unicode", "value": "UTF_16_BE" }, - { "type": "unicode", "value": "UTF_16_LE" }, - { "type": "unicode", "value": "UTF_32" }, - { "type": "unicode", "value": "UTF_32_BE" }, - { "type": "unicode", "value": "UTF_32_LE" }, - { "type": "unicode", "value": "UTF_7" }, - { "type": "unicode", "value": "UTF_8" }, - { "type": "unicode", "value": "UTF_8_SIG" }, - { "type": "unicode", "value": "UU_CODEC" }, - { "type": "unicode", "value": "ZLIB_CODEC" } + { + "type": "unicode", + "value": "BASE64_CODEC" + }, + { + "type": "unicode", + "value": "BIG5" + }, + { + "type": "unicode", + "value": "BIG5HKSCS" + }, + { + "type": "unicode", + "value": "CP037" + }, + { + "type": "unicode", + "value": "CP1006" + }, + { + "type": "unicode", + "value": "CP1026" + }, + { + "type": "unicode", + "value": "CP1140" + }, + { + "type": "unicode", + "value": "CP1250" + }, + { + "type": "unicode", + "value": "CP1251" + }, + { + "type": "unicode", + "value": "CP1252" + }, + { + "type": "unicode", + "value": "CP1253" + }, + { + "type": "unicode", + "value": "CP1254" + }, + { + "type": "unicode", + "value": "CP1255" + }, + { + "type": "unicode", + "value": "CP1256" + }, + { + "type": "unicode", + "value": "CP1257" + }, + { + "type": "unicode", + "value": "CP1258" + }, + { + "type": "unicode", + "value": "CP424" + }, + { + "type": "unicode", + "value": "CP437" + }, + { + "type": "unicode", + "value": "CP500" + }, + { + "type": "unicode", + "value": "CP737" + }, + { + "type": "unicode", + "value": "CP775" + }, + { + "type": "unicode", + "value": "CP850" + }, + { + "type": "unicode", + "value": "CP852" + }, + { + "type": "unicode", + "value": "CP855" + }, + { + "type": "unicode", + "value": "CP856" + }, + { + "type": "unicode", + "value": "CP857" + }, + { + "type": "unicode", + "value": "CP860" + }, + { + "type": "unicode", + "value": "CP861" + }, + { + "type": "unicode", + "value": "CP862" + }, + { + "type": "unicode", + "value": "CP863" + }, + { + "type": "unicode", + "value": "CP864" + }, + { + "type": "unicode", + "value": "CP865" + }, + { + "type": "unicode", + "value": "CP866" + }, + { + "type": "unicode", + "value": "CP869" + }, + { + "type": "unicode", + "value": "CP874" + }, + { + "type": "unicode", + "value": "CP875" + }, + { + "type": "unicode", + "value": "CP932" + }, + { + "type": "unicode", + "value": "CP949" + }, + { + "type": "unicode", + "value": "CP950" + }, + { + "type": "unicode", + "value": "IDNA" + }, + { + "type": "unicode", + "value": "ROT_13" + }, + { + "type": "unicode", + "value": "UTF_16" + }, + { + "type": "unicode", + "value": "UTF_16_BE" + }, + { + "type": "unicode", + "value": "UTF_16_LE" + }, + { + "type": "unicode", + "value": "UTF_32" + }, + { + "type": "unicode", + "value": "UTF_32_BE" + }, + { + "type": "unicode", + "value": "UTF_32_LE" + }, + { + "type": "unicode", + "value": "UTF_7" + }, + { + "type": "unicode", + "value": "UTF_8" + }, + { + "type": "unicode", + "value": "UTF_8_SIG" + }, + { + "type": "unicode", + "value": "UU_CODEC" + }, + { + "type": "unicode", + "value": "ZLIB_CODEC" + } ] }, "test_class": "ApiListKnownEncodingsHandlerRegressionTest_http_v1", @@ -11756,7 +17374,10 @@ { "type": "ApiNotification", "value": { - "is_pending": { "type": "bool", "value": true }, + "is_pending": { + "type": "bool", + "value": true + }, "message": { "type": "unicode", "value": "Host-0.example.com: " @@ -11777,16 +17398,25 @@ } } }, - "type": { "type": "EnumNamedValue", "value": "CLIENT" } + "type": { + "type": "EnumNamedValue", + "value": "CLIENT" + } } }, - "timestamp": { "type": "RDFDatetime", "value": 42000000 } + "timestamp": { + "type": "RDFDatetime", + "value": 42000000 + } } }, { "type": "ApiNotification", "value": { - "is_pending": { "type": "bool", "value": true }, + "is_pending": { + "type": "bool", + "value": true + }, "message": { "type": "unicode", "value": "Host-0.example.com: " @@ -11798,7 +17428,10 @@ "reference": { "type": "ApiNotificationReference", "value": { - "type": { "type": "EnumNamedValue", "value": "VFS" }, + "type": { + "type": "EnumNamedValue", + "value": "VFS" + }, "vfs": { "type": "ApiNotificationVfsReference", "value": { @@ -11806,12 +17439,18 @@ "type": "ApiClientId", "value": "C.1000000000000000" }, - "vfs_path": { "type": "unicode", "value": "fs/os/foo" } + "vfs_path": { + "type": "unicode", + "value": "fs/os/foo" + } } } } }, - "timestamp": { "type": "RDFDatetime", "value": 44000000 } + "timestamp": { + "type": "RDFDatetime", + "value": 44000000 + } } } ] @@ -11824,7 +17463,9 @@ "message": "Host-0.example.com: ", "notification_type": "TYPE_CLIENT_INTERROGATED", "reference": { - "client": { "client_id": "C.1000000000000000" }, + "client": { + "client_id": "C.1000000000000000" + }, "type": "CLIENT" }, "timestamp": 42000000 @@ -11854,7 +17495,10 @@ { "type": "ApiNotification", "value": { - "is_pending": { "type": "bool", "value": true }, + "is_pending": { + "type": "bool", + "value": true + }, "message": { "type": "unicode", "value": "Host-0.example.com: " @@ -11866,7 +17510,10 @@ "reference": { "type": "ApiNotificationReference", "value": { - "type": { "type": "EnumNamedValue", "value": "VFS" }, + "type": { + "type": "EnumNamedValue", + "value": "VFS" + }, "vfs": { "type": "ApiNotificationVfsReference", "value": { @@ -11874,12 +17521,18 @@ "type": "ApiClientId", "value": "C.1000000000000000" }, - "vfs_path": { "type": "unicode", "value": "fs/os/foo" } + "vfs_path": { + "type": "unicode", + "value": "fs/os/foo" + } } } } }, - "timestamp": { "type": "RDFDatetime", "value": 44000000 } + "timestamp": { + "type": "RDFDatetime", + "value": 44000000 + } } } ] @@ -11907,7 +17560,9 @@ { "api_method": "ListPendingUserNotifications", "method": "GET", - "response": { "items": [] }, + "response": { + "items": [] + }, "test_class": "ApiListPendingUserNotificationsHandlerRegressionTest_http_v1", "url": "/api/users/me/notifications/pending?timestamp=44000000" } @@ -11924,14 +17579,26 @@ "desc": { "type": "ApiReportDescriptor", "value": { - "name": { "type": "unicode", "value": "FooReportPlugin" }, - "requires_time_range": { "type": "bool", "value": false }, + "name": { + "type": "unicode", + "value": "FooReportPlugin" + }, + "requires_time_range": { + "type": "bool", + "value": false + }, "summary": { "type": "unicode", "value": "Reports all foos." }, - "title": { "type": "unicode", "value": "Foo" }, - "type": { "type": "EnumNamedValue", "value": "CLIENT" } + "title": { + "type": "unicode", + "value": "Foo" + }, + "type": { + "type": "EnumNamedValue", + "value": "CLIENT" + } } } } @@ -11942,14 +17609,26 @@ "desc": { "type": "ApiReportDescriptor", "value": { - "name": { "type": "unicode", "value": "BarReportPlugin" }, - "requires_time_range": { "type": "bool", "value": true }, + "name": { + "type": "unicode", + "value": "BarReportPlugin" + }, + "requires_time_range": { + "type": "bool", + "value": true + }, "summary": { "type": "unicode", "value": "Reports bars' activity in the given time range." }, - "title": { "type": "unicode", "value": "Bar Activity" }, - "type": { "type": "EnumNamedValue", "value": "SERVER" } + "title": { + "type": "unicode", + "value": "Bar Activity" + }, + "type": { + "type": "EnumNamedValue", + "value": "SERVER" + } } } } @@ -11986,11 +17665,16 @@ { "api_method": "ModifyCronJob", "method": "PATCH", - "request_payload": { "enabled": true }, + "request_payload": { + "enabled": true + }, "response": { "type": "ApiCronJob", "value": { - "allow_overruns": { "type": "bool", "value": false }, + "allow_overruns": { + "type": "bool", + "value": false + }, "args": { "type": "CronJobAction", "value": { @@ -12001,8 +17685,14 @@ "hunt_cron_action": { "type": "HuntCronAction", "value": { - "flow_args": { "type": "FileFinderArgs", "value": {} }, - "flow_name": { "type": "unicode", "value": "FileFinder" }, + "flow_args": { + "type": "FileFinderArgs", + "value": {} + }, + "flow_name": { + "type": "unicode", + "value": "FileFinder" + }, "hunt_runner_args": { "type": "HuntRunnerArgs", "value": { @@ -12018,20 +17708,44 @@ "type": "long", "value": 1000 }, - "client_rate": { "type": "float", "value": 20.0 }, - "crash_limit": { "type": "long", "value": 100 } + "client_rate": { + "type": "float", + "value": 20.0 + }, + "crash_limit": { + "type": "long", + "value": 100 + } } } } } } }, - "cron_job_id": { "type": "ApiCronJobId", "value": "FileFinder" }, - "description": { "type": "unicode", "value": "" }, - "enabled": { "type": "bool", "value": true }, - "frequency": { "type": "DurationSeconds", "value": 86400 }, - "is_failing": { "type": "bool", "value": false }, - "lifetime": { "type": "DurationSeconds", "value": 604800 } + "cron_job_id": { + "type": "ApiCronJobId", + "value": "FileFinder" + }, + "description": { + "type": "unicode", + "value": "" + }, + "enabled": { + "type": "bool", + "value": true + }, + "frequency": { + "type": "DurationSeconds", + "value": 86400 + }, + "is_failing": { + "type": "bool", + "value": false + }, + "lifetime": { + "type": "DurationSeconds", + "value": 604800 + } } }, "test_class": "ApiModifyCronJobRegressionTest_http_v1", @@ -12063,11 +17777,16 @@ { "api_method": "ModifyCronJob", "method": "PATCH", - "request_payload": { "enabled": false }, + "request_payload": { + "enabled": false + }, "response": { "type": "ApiCronJob", "value": { - "allow_overruns": { "type": "bool", "value": false }, + "allow_overruns": { + "type": "bool", + "value": false + }, "args": { "type": "CronJobAction", "value": { @@ -12078,7 +17797,10 @@ "hunt_cron_action": { "type": "HuntCronAction", "value": { - "flow_args": { "type": "FileFinderArgs", "value": {} }, + "flow_args": { + "type": "FileFinderArgs", + "value": {} + }, "flow_name": { "type": "unicode", "value": "ClientFileFinder" @@ -12098,8 +17820,14 @@ "type": "long", "value": 1000 }, - "client_rate": { "type": "float", "value": 20.0 }, - "crash_limit": { "type": "long", "value": 100 } + "client_rate": { + "type": "float", + "value": 20.0 + }, + "crash_limit": { + "type": "long", + "value": 100 + } } } } @@ -12110,11 +17838,26 @@ "type": "ApiCronJobId", "value": "ClientFileFinder" }, - "description": { "type": "unicode", "value": "" }, - "enabled": { "type": "bool", "value": false }, - "frequency": { "type": "DurationSeconds", "value": 86400 }, - "is_failing": { "type": "bool", "value": false }, - "lifetime": { "type": "DurationSeconds", "value": 604800 } + "description": { + "type": "unicode", + "value": "" + }, + "enabled": { + "type": "bool", + "value": false + }, + "frequency": { + "type": "DurationSeconds", + "value": 86400 + }, + "is_failing": { + "type": "bool", + "value": false + }, + "lifetime": { + "type": "DurationSeconds", + "value": 604800 + } } }, "test_class": "ApiModifyCronJobRegressionTest_http_v1", @@ -12148,13 +17891,24 @@ { "api_method": "ModifyHunt", "method": "PATCH", - "request_payload": { "client_limit": 142 }, + "request_payload": { + "client_limit": 142 + }, "response": { "type": "ApiHunt", "value": { - "all_clients_count": { "type": "long", "value": 0 }, - "client_limit": { "type": "long", "value": 142 }, - "client_rate": { "type": "float", "value": 0.0 }, + "all_clients_count": { + "type": "long", + "value": 0 + }, + "client_limit": { + "type": "long", + "value": 142 + }, + "client_rate": { + "type": "float", + "value": 0.0 + }, "client_rule_set": { "type": "ForemanClientRuleSet", "value": { @@ -12175,35 +17929,77 @@ } } }, - "rule_type": { "type": "EnumNamedValue", "value": "REGEX" } + "rule_type": { + "type": "EnumNamedValue", + "value": "REGEX" + } } } ] } }, - "clients_with_results_count": { "type": "long", "value": 0 }, - "completed_clients_count": { "type": "long", "value": 0 }, - "crash_limit": { "type": "long", "value": 100 }, - "crashed_clients_count": { "type": "long", "value": 0 }, - "created": { "type": "RDFDatetime", "value": 42000000 }, - "creator": { "type": "unicode", "value": "api_test_user" }, - "description": { "type": "unicode", "value": "the hunt" }, - "duration": { "type": "DurationSeconds", "value": 1209600 }, - "failed_clients_count": { "type": "long", "value": 0 }, + "clients_with_results_count": { + "type": "long", + "value": 0 + }, + "completed_clients_count": { + "type": "long", + "value": 0 + }, + "crash_limit": { + "type": "long", + "value": 100 + }, + "crashed_clients_count": { + "type": "long", + "value": 0 + }, + "created": { + "type": "RDFDatetime", + "value": 42000000 + }, + "creator": { + "type": "unicode", + "value": "api_test_user" + }, + "description": { + "type": "unicode", + "value": "the hunt" + }, + "duration": { + "type": "DurationSeconds", + "value": 1209600 + }, + "failed_clients_count": { + "type": "long", + "value": 0 + }, "flow_args": { "type": "GetFileArgs", "value": { "pathspec": { "type": "PathSpec", "value": { - "path": { "type": "unicode", "value": "/tmp/evil.txt" }, - "pathtype": { "type": "EnumNamedValue", "value": "OS" } + "path": { + "type": "unicode", + "value": "/tmp/evil.txt" + }, + "pathtype": { + "type": "EnumNamedValue", + "value": "OS" + } } } } }, - "flow_name": { "type": "unicode", "value": "GetFile" }, - "hunt_id": { "type": "ApiHuntId", "value": "H:123456" }, + "flow_name": { + "type": "unicode", + "value": "GetFile" + }, + "hunt_id": { + "type": "ApiHuntId", + "value": "H:123456" + }, "hunt_runner_args": { "type": "HuntRunnerArgs", "value": { @@ -12215,9 +18011,18 @@ "type": "long", "value": 10485760 }, - "avg_results_per_client_limit": { "type": "long", "value": 1000 }, - "client_limit": { "type": "long", "value": 142 }, - "client_rate": { "type": "float", "value": 0.0 }, + "avg_results_per_client_limit": { + "type": "long", + "value": 1000 + }, + "client_limit": { + "type": "long", + "value": 142 + }, + "client_rate": { + "type": "float", + "value": 0.0 + }, "client_rule_set": { "type": "ForemanClientRuleSet", "value": { @@ -12247,26 +18052,72 @@ ] } }, - "crash_limit": { "type": "long", "value": 100 }, - "description": { "type": "unicode", "value": "the hunt" }, - "expiry_time": { "type": "DurationSeconds", "value": 1209600 }, - "hunt_name": { "type": "unicode", "value": "GenericHunt" }, + "crash_limit": { + "type": "long", + "value": 100 + }, + "description": { + "type": "unicode", + "value": "the hunt" + }, + "expiry_time": { + "type": "DurationSeconds", + "value": 1209600 + }, + "hunt_name": { + "type": "unicode", + "value": "GenericHunt" + }, "original_object": { "type": "FlowLikeObjectReference", "value": {} } } }, - "hunt_type": { "type": "EnumNamedValue", "value": "STANDARD" }, - "is_robot": { "type": "bool", "value": false }, - "name": { "type": "unicode", "value": "GenericHunt" }, - "remaining_clients_count": { "type": "long", "value": 0 }, - "results_count": { "type": "long", "value": 0 }, - "state": { "type": "EnumNamedValue", "value": "PAUSED" }, - "state_comment": { "type": "unicode", "value": "" }, - "total_cpu_usage": { "type": "long", "value": 0 }, - "total_net_usage": { "type": "long", "value": 0 }, - "urn": { "type": "SessionID", "value": "aff4:/hunts/H:123456" } + "hunt_type": { + "type": "EnumNamedValue", + "value": "STANDARD" + }, + "is_robot": { + "type": "bool", + "value": false + }, + "name": { + "type": "unicode", + "value": "GenericHunt" + }, + "remaining_clients_count": { + "type": "long", + "value": 0 + }, + "results_count": { + "type": "long", + "value": 0 + }, + "state": { + "type": "EnumNamedValue", + "value": "PAUSED" + }, + "state_comment": { + "type": "unicode", + "value": "" + }, + "state_reason": { + "type": "EnumNamedValue", + "value": "UNKNOWN" + }, + "total_cpu_usage": { + "type": "long", + "value": 0 + }, + "total_net_usage": { + "type": "long", + "value": 0 + }, + "urn": { + "type": "SessionID", + "value": "aff4:/hunts/H:123456" + } } }, "test_class": "ApiModifyHuntHandlerRegressionTest_http_v1", @@ -12277,7 +18128,10 @@ "client_rule_set": { "rules": [ { - "regex": { "attribute_regex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attribute_regex": "GRR", + "field": "CLIENT_NAME" + }, "rule_type": "REGEX" } ] @@ -12292,7 +18146,10 @@ "duration": 1209600, "failed_clients_count": 0, "flow_args": { - "pathspec": { "path": "/tmp/evil.txt", "pathtype": "OS" } + "pathspec": { + "path": "/tmp/evil.txt", + "pathtype": "OS" + } }, "flow_name": "GetFile", "hunt_id": "H:123456", @@ -12305,7 +18162,10 @@ "client_rule_set": { "rules": [ { - "regex": { "attribute_regex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attribute_regex": "GRR", + "field": "CLIENT_NAME" + }, "rule_type": "REGEX" } ] @@ -12323,6 +18183,7 @@ "results_count": 0, "state": "PAUSED", "state_comment": "", + "state_reason": "UNKNOWN", "total_cpu_usage": 0, "total_net_usage": 0, "urn": "aff4:/hunts/H:123456" @@ -12332,13 +18193,24 @@ { "api_method": "ModifyHunt", "method": "PATCH", - "request_payload": { "state": "STOPPED" }, + "request_payload": { + "state": "STOPPED" + }, "response": { "type": "ApiHunt", "value": { - "all_clients_count": { "type": "long", "value": 0 }, - "client_limit": { "type": "long", "value": 142 }, - "client_rate": { "type": "float", "value": 0.0 }, + "all_clients_count": { + "type": "long", + "value": 0 + }, + "client_limit": { + "type": "long", + "value": 142 + }, + "client_rate": { + "type": "float", + "value": 0.0 + }, "client_rule_set": { "type": "ForemanClientRuleSet", "value": { @@ -12359,35 +18231,77 @@ } } }, - "rule_type": { "type": "EnumNamedValue", "value": "REGEX" } + "rule_type": { + "type": "EnumNamedValue", + "value": "REGEX" + } } } ] } }, - "clients_with_results_count": { "type": "long", "value": 0 }, - "completed_clients_count": { "type": "long", "value": 0 }, - "crash_limit": { "type": "long", "value": 100 }, - "crashed_clients_count": { "type": "long", "value": 0 }, - "created": { "type": "RDFDatetime", "value": 42000000 }, - "creator": { "type": "unicode", "value": "api_test_user" }, - "description": { "type": "unicode", "value": "the hunt" }, - "duration": { "type": "DurationSeconds", "value": 1209600 }, - "failed_clients_count": { "type": "long", "value": 0 }, + "clients_with_results_count": { + "type": "long", + "value": 0 + }, + "completed_clients_count": { + "type": "long", + "value": 0 + }, + "crash_limit": { + "type": "long", + "value": 100 + }, + "crashed_clients_count": { + "type": "long", + "value": 0 + }, + "created": { + "type": "RDFDatetime", + "value": 42000000 + }, + "creator": { + "type": "unicode", + "value": "api_test_user" + }, + "description": { + "type": "unicode", + "value": "the hunt" + }, + "duration": { + "type": "DurationSeconds", + "value": 1209600 + }, + "failed_clients_count": { + "type": "long", + "value": 0 + }, "flow_args": { "type": "GetFileArgs", "value": { "pathspec": { "type": "PathSpec", "value": { - "path": { "type": "unicode", "value": "/tmp/evil.txt" }, - "pathtype": { "type": "EnumNamedValue", "value": "OS" } + "path": { + "type": "unicode", + "value": "/tmp/evil.txt" + }, + "pathtype": { + "type": "EnumNamedValue", + "value": "OS" + } } } } }, - "flow_name": { "type": "unicode", "value": "GetFile" }, - "hunt_id": { "type": "ApiHuntId", "value": "H:123456" }, + "flow_name": { + "type": "unicode", + "value": "GetFile" + }, + "hunt_id": { + "type": "ApiHuntId", + "value": "H:123456" + }, "hunt_runner_args": { "type": "HuntRunnerArgs", "value": { @@ -12399,9 +18313,18 @@ "type": "long", "value": 10485760 }, - "avg_results_per_client_limit": { "type": "long", "value": 1000 }, - "client_limit": { "type": "long", "value": 142 }, - "client_rate": { "type": "float", "value": 0.0 }, + "avg_results_per_client_limit": { + "type": "long", + "value": 1000 + }, + "client_limit": { + "type": "long", + "value": 142 + }, + "client_rate": { + "type": "float", + "value": 0.0 + }, "client_rule_set": { "type": "ForemanClientRuleSet", "value": { @@ -12431,26 +18354,72 @@ ] } }, - "crash_limit": { "type": "long", "value": 100 }, - "description": { "type": "unicode", "value": "the hunt" }, - "expiry_time": { "type": "DurationSeconds", "value": 1209600 }, - "hunt_name": { "type": "unicode", "value": "GenericHunt" }, + "crash_limit": { + "type": "long", + "value": 100 + }, + "description": { + "type": "unicode", + "value": "the hunt" + }, + "expiry_time": { + "type": "DurationSeconds", + "value": 1209600 + }, + "hunt_name": { + "type": "unicode", + "value": "GenericHunt" + }, "original_object": { "type": "FlowLikeObjectReference", "value": {} } } }, - "hunt_type": { "type": "EnumNamedValue", "value": "STANDARD" }, - "is_robot": { "type": "bool", "value": false }, - "name": { "type": "unicode", "value": "GenericHunt" }, - "remaining_clients_count": { "type": "long", "value": 0 }, - "results_count": { "type": "long", "value": 0 }, - "state": { "type": "EnumNamedValue", "value": "STOPPED" }, - "state_comment": { "type": "unicode", "value": "Cancelled by user" }, - "total_cpu_usage": { "type": "long", "value": 0 }, - "total_net_usage": { "type": "long", "value": 0 }, - "urn": { "type": "SessionID", "value": "aff4:/hunts/H:123456" } + "hunt_type": { + "type": "EnumNamedValue", + "value": "STANDARD" + }, + "is_robot": { + "type": "bool", + "value": false + }, + "name": { + "type": "unicode", + "value": "GenericHunt" + }, + "remaining_clients_count": { + "type": "long", + "value": 0 + }, + "results_count": { + "type": "long", + "value": 0 + }, + "state": { + "type": "EnumNamedValue", + "value": "STOPPED" + }, + "state_comment": { + "type": "unicode", + "value": "Cancelled by user" + }, + "state_reason": { + "type": "EnumNamedValue", + "value": "TRIGGERED_BY_USER" + }, + "total_cpu_usage": { + "type": "long", + "value": 0 + }, + "total_net_usage": { + "type": "long", + "value": 0 + }, + "urn": { + "type": "SessionID", + "value": "aff4:/hunts/H:123456" + } } }, "test_class": "ApiModifyHuntHandlerRegressionTest_http_v1", @@ -12461,7 +18430,10 @@ "client_rule_set": { "rules": [ { - "regex": { "attribute_regex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attribute_regex": "GRR", + "field": "CLIENT_NAME" + }, "rule_type": "REGEX" } ] @@ -12476,7 +18448,10 @@ "duration": 1209600, "failed_clients_count": 0, "flow_args": { - "pathspec": { "path": "/tmp/evil.txt", "pathtype": "OS" } + "pathspec": { + "path": "/tmp/evil.txt", + "pathtype": "OS" + } }, "flow_name": "GetFile", "hunt_id": "H:123456", @@ -12489,7 +18464,10 @@ "client_rule_set": { "rules": [ { - "regex": { "attribute_regex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attribute_regex": "GRR", + "field": "CLIENT_NAME" + }, "rule_type": "REGEX" } ] @@ -12507,6 +18485,7 @@ "results_count": 0, "state": "STOPPED", "state_comment": "Cancelled by user", + "state_reason": "TRIGGERED_BY_USER", "total_cpu_usage": 0, "total_net_usage": 0, "urn": "aff4:/hunts/H:123456" @@ -12523,16 +18502,34 @@ { "type": "ApiClient", "value": { - "age": { "type": "RDFDatetime", "value": 42000000 }, + "age": { + "type": "RDFDatetime", + "value": 42000000 + }, "agent_info": { "type": "ClientInformation", "value": { - "build_time": { "type": "unicode", "value": "1980-01-01" }, - "client_name": { "type": "unicode", "value": "GRR Monitor" }, - "client_version": { "type": "long", "value": 1234 }, + "build_time": { + "type": "unicode", + "value": "1980-01-01" + }, + "client_name": { + "type": "unicode", + "value": "GRR Monitor" + }, + "client_version": { + "type": "long", + "value": 1234 + }, "labels": [ - { "type": "unicode", "value": "label1" }, - { "type": "unicode", "value": "label2" } + { + "type": "unicode", + "value": "label1" + }, + { + "type": "unicode", + "value": "label2" + } ] } }, @@ -12540,7 +18537,6 @@ "type": "ApiClientId", "value": "C.1000000000000000" }, - "fleetspeak_enabled": { "type": "bool", "value": false }, "hardware_info": { "type": "HardwareInfo", "value": { @@ -12586,55 +18582,100 @@ } } ], - "ifname": { "type": "unicode", "value": "if0" } + "ifname": { + "type": "unicode", + "value": "if0" + } } }, { "type": "Interface", "value": { - "ifname": { "type": "unicode", "value": "if1" }, - "mac_address": { "type": "MacAddress", "value": "qrvM3e4A" } + "ifname": { + "type": "unicode", + "value": "if1" + }, + "mac_address": { + "type": "MacAddress", + "value": "qrvM3e4A" + } } }, { "type": "Interface", "value": { - "ifname": { "type": "unicode", "value": "if2" }, - "mac_address": { "type": "MacAddress", "value": "u8zd7v8A" } + "ifname": { + "type": "unicode", + "value": "if2" + }, + "mac_address": { + "type": "MacAddress", + "value": "u8zd7v8A" + } } } ], "knowledge_base": { "type": "KnowledgeBase", "value": { - "fqdn": { "type": "unicode", "value": "Host-0.example.com" }, - "os": { "type": "unicode", "value": "Linux" }, + "fqdn": { + "type": "unicode", + "value": "Host-0.example.com" + }, + "os": { + "type": "unicode", + "value": "Linux" + }, "users": [ { "type": "User", "value": { - "username": { "type": "unicode", "value": "user1" } + "username": { + "type": "unicode", + "value": "user1" + } } }, { "type": "User", "value": { - "username": { "type": "unicode", "value": "user2" } + "username": { + "type": "unicode", + "value": "user2" + } } } ] } }, "labels": [], - "last_seen_at": { "type": "RDFDatetime", "value": 42000000 }, + "last_seen_at": { + "type": "RDFDatetime", + "value": 42000000 + }, "os_info": { "type": "Uname", "value": { - "fqdn": { "type": "unicode", "value": "Host-0.example.com" }, - "kernel": { "type": "unicode", "value": "4.0.0" }, - "machine": { "type": "unicode", "value": "x86_64" }, - "system": { "type": "unicode", "value": "Linux" }, - "version": { "type": "unicode", "value": "buster/sid" } + "fqdn": { + "type": "unicode", + "value": "Host-0.example.com" + }, + "kernel": { + "type": "unicode", + "value": "4.0.0" + }, + "machine": { + "type": "unicode", + "value": "x86_64" + }, + "system": { + "type": "unicode", + "value": "Linux" + }, + "version": { + "type": "unicode", + "value": "buster/sid" + } } }, "urn": { @@ -12645,13 +18686,19 @@ { "type": "User", "value": { - "username": { "type": "unicode", "value": "user1" } + "username": { + "type": "unicode", + "value": "user1" + } } }, { "type": "User", "value": { - "username": { "type": "unicode", "value": "user2" } + "username": { + "type": "unicode", + "value": "user2" + } } } ] @@ -12668,10 +18715,12 @@ "build_time": "1980-01-01", "client_name": "GRR Monitor", "client_version": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "client_id": "C.1000000000000000", - "fleetspeak_enabled": false, "hardware_info": { "bios_version": "Bios-Version-0", "system_manufacturer": "System-Manufacturer-0" @@ -12679,7 +18728,10 @@ "interfaces": [ { "addresses": [ - { "address_type": "INET", "packed_bytes": "wKgAAA==" }, + { + "address_type": "INET", + "packed_bytes": "wKgAAA==" + }, { "address_type": "INET6", "packed_bytes": "IAGrzQAAAAAAAAAAAAAAAA==" @@ -12687,13 +18739,26 @@ ], "ifname": "if0" }, - { "ifname": "if1", "mac_address": "qrvM3e4A" }, - { "ifname": "if2", "mac_address": "u8zd7v8A" } + { + "ifname": "if1", + "mac_address": "qrvM3e4A" + }, + { + "ifname": "if2", + "mac_address": "u8zd7v8A" + } ], "knowledge_base": { "fqdn": "Host-0.example.com", "os": "Linux", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] }, "labels": [], "last_seen_at": 42000000, @@ -12705,7 +18770,14 @@ "version": "buster/sid" }, "urn": "aff4:/C.1000000000000000", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] } ] }, @@ -12716,10 +18788,14 @@ { "api_method": "UpdateVfsFileContent", "method": "POST", - "request_payload": { "file_path": "fs/os/c/bin/bash" }, - "response": { "operation_id": "ABCDEF" }, + "request_payload": { + "file_path": "fs/os/c/bin/bash" + }, + "response": { + "operation_id": "ABCDEF" + }, "test_class": "ApiUpdateVfsFileContentHandlerRegressionTest_http_v1", "url": "/api/clients/C.1000000000000000/vfs-update" } ] -} +} \ No newline at end of file diff --git a/grr/server/grr_response_server/gui/static/angular-components/docs/api-v2-docs-examples.json b/grr/server/grr_response_server/gui/static/angular-components/docs/api-v2-docs-examples.json index e4398cfb5c..1b2619e5c9 100644 --- a/grr/server/grr_response_server/gui/static/angular-components/docs/api-v2-docs-examples.json +++ b/grr/server/grr_response_server/gui/static/angular-components/docs/api-v2-docs-examples.json @@ -31,7 +31,14 @@ { "api_method": "CountHuntResultsByType", "method": "GET", - "response": { "items": [{ "count": "1", "type": "RDFString" }] }, + "response": { + "items": [ + { + "count": "1", + "type": "RDFString" + } + ] + }, "test_class": "ApiCountHuntResultsByTypeRegressionTest_http_v2", "url": "/api/v2/hunts/H:123456/result-counts" } @@ -42,20 +49,32 @@ "method": "POST", "request_payload": { "approval": { - "emailCcAddresses": ["test@example.com"], - "notifiedUsers": ["approver1", "approver2"], + "emailCcAddresses": [ + "test@example.com" + ], + "notifiedUsers": [ + "approver1", + "approver2" + ], "reason": "really important reason!" } }, "response": { - "approvers": ["api_test_user"], - "emailCcAddresses": ["test@example.com"], + "approvers": [ + "api_test_user" + ], + "emailCcAddresses": [ + "test@example.com" + ], "emailMessageId": "", "expirationTimeUs": "2419326000000", "id": "approval:112233", "isValid": false, "isValidMessage": "Need at least 1 additional approver for access.", - "notifiedUsers": ["approver1", "approver2"], + "notifiedUsers": [ + "approver1", + "approver2" + ], "reason": "really important reason!", "requestor": "api_test_user", "subject": { @@ -64,10 +83,12 @@ "buildTime": "1980-01-01", "clientName": "GRR Monitor", "clientVersion": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "clientId": "C.1000000000000000", - "fleetspeakEnabled": false, "hardwareInfo": { "biosVersion": "Bios-Version-0", "systemManufacturer": "System-Manufacturer-0" @@ -75,7 +96,10 @@ "interfaces": [ { "addresses": [ - { "addressType": "INET", "packedBytes": "wKgAAA==" }, + { + "addressType": "INET", + "packedBytes": "wKgAAA==" + }, { "addressType": "INET6", "packedBytes": "IAGrzQAAAAAAAAAAAAAAAA==" @@ -83,13 +107,26 @@ ], "ifname": "if0" }, - { "ifname": "if1", "macAddress": "qrvM3e4A" }, - { "ifname": "if2", "macAddress": "u8zd7v8A" } + { + "ifname": "if1", + "macAddress": "qrvM3e4A" + }, + { + "ifname": "if2", + "macAddress": "u8zd7v8A" + } ], "knowledgeBase": { "fqdn": "Host-0.example.com", "os": "Linux", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] }, "lastSeenAt": "42000000", "osInfo": { @@ -100,7 +137,14 @@ "version": "buster/sid" }, "urn": "aff4:/C.1000000000000000", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] } }, "test_class": "ApiCreateClientApprovalHandlerRegressionTest_http_v2", @@ -113,19 +157,31 @@ "method": "POST", "request_payload": { "approval": { - "emailCcAddresses": ["test@example.com"], - "notifiedUsers": ["approver1", "approver2"], + "emailCcAddresses": [ + "test@example.com" + ], + "notifiedUsers": [ + "approver1", + "approver2" + ], "reason": "really important reason!" } }, "response": { - "approvers": ["api_test_user"], - "emailCcAddresses": ["test@example.com"], + "approvers": [ + "api_test_user" + ], + "emailCcAddresses": [ + "test@example.com" + ], "emailMessageId": "", "id": "approval:112233", "isValid": false, "isValidMessage": "Need at least 1 additional approver for access.", - "notifiedUsers": ["approver1", "approver2"], + "notifiedUsers": [ + "approver1", + "approver2" + ], "reason": "really important reason!", "requestor": "api_test_user", "subject": { @@ -167,7 +223,9 @@ "filenameRegex": "." }, "name": "ListProcesses", - "runnerArgs": { "notifyToUser": true } + "runnerArgs": { + "notifyToUser": true + } } }, "response": { @@ -201,20 +259,32 @@ "method": "POST", "request_payload": { "approval": { - "emailCcAddresses": ["test@example.com"], - "notifiedUsers": ["approver1", "approver2"], + "emailCcAddresses": [ + "test@example.com" + ], + "notifiedUsers": [ + "approver1", + "approver2" + ], "reason": "really important reason!" } }, "response": { - "approvers": ["api_test_user"], - "emailCcAddresses": ["test@example.com"], + "approvers": [ + "api_test_user" + ], + "emailCcAddresses": [ + "test@example.com" + ], "emailMessageId": "", "expirationTimeUs": "2419326000000", "id": "approval:112233", "isValid": false, "isValidMessage": "Need at least 1 additional approver for access.", - "notifiedUsers": ["approver1", "approver2"], + "notifiedUsers": [ + "approver1", + "approver2" + ], "reason": "really important reason!", "requestor": "api_test_user", "subject": { @@ -224,7 +294,10 @@ "clientRuleSet": { "rules": [ { - "regex": { "attributeRegex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attributeRegex": "GRR", + "field": "CLIENT_NAME" + }, "ruleType": "REGEX" } ] @@ -240,7 +313,10 @@ "failedClientsCount": "0", "flowArgs": { "@type": "type.googleapis.com/grr.GetFileArgs", - "pathspec": { "path": "/tmp/evil.txt", "pathtype": "OS" } + "pathspec": { + "path": "/tmp/evil.txt", + "pathtype": "OS" + } }, "flowName": "GetFile", "huntId": "H:123456", @@ -252,7 +328,10 @@ "clientRuleSet": { "rules": [ { - "regex": { "attributeRegex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attributeRegex": "GRR", + "field": "CLIENT_NAME" + }, "ruleType": "REGEX" } ] @@ -270,6 +349,7 @@ "resultsCount": "0", "state": "PAUSED", "stateComment": "", + "stateReason": "UNKNOWN", "totalCpuUsage": 0.0, "totalNetUsage": "0", "urn": "aff4:/hunts/H:123456" @@ -288,7 +368,10 @@ "perClientArgs": [ { "clientId": "C.1000000000000000", - "paths": ["/etc/hosts", "/foo/bar"] + "paths": [ + "/etc/hosts", + "/foo/bar" + ] } ] }, @@ -309,12 +392,20 @@ "@type": "type.googleapis.com/grr.HuntArgumentsVariable", "flowGroups": [ { - "clientIds": ["C.1000000000000000"], + "clientIds": [ + "C.1000000000000000" + ], "flowArgs": { "@type": "type.googleapis.com/grr.MultiGetFileArgs", "pathspecs": [ - { "path": "/etc/hosts", "pathtype": "OS" }, - { "path": "/foo/bar", "pathtype": "OS" } + { + "path": "/etc/hosts", + "pathtype": "OS" + }, + { + "path": "/foo/bar", + "pathtype": "OS" + } ] }, "flowName": "MultiGetFile" @@ -341,6 +432,7 @@ "resultsCount": "0", "state": "PAUSED", "stateComment": "", + "stateReason": "UNKNOWN", "totalCpuUsage": 0.0, "totalNetUsage": "0", "urn": "aff4:/hunts/H:123456" @@ -353,8 +445,13 @@ { "api_method": "CreateVfsRefreshOperation", "method": "POST", - "request_payload": { "filePath": "fs/os/Users/Shared", "maxDepth": "1" }, - "response": { "operationId": "ABCDEF" }, + "request_payload": { + "filePath": "fs/os/Users/Shared", + "maxDepth": "1" + }, + "response": { + "operationId": "ABCDEF" + }, "test_class": "ApiCreateVfsRefreshOperationHandlerRegressionTest_http_v2", "url": "/api/v2/clients/C.1000000000000000/vfs-refresh-operations" } @@ -363,7 +460,9 @@ { "api_method": "DeleteHunt", "method": "DELETE", - "response": { "status": "OK" }, + "response": { + "status": "OK" + }, "test_class": "ApiDeleteHuntHandlerRegressionTest_http_v2", "url": "/api/v2/hunts/H:123456" } @@ -372,9 +471,18 @@ { "api_method": "ExplainGlobExpression", "method": "POST", - "request_payload": { "globExpression": "/foo/*" }, + "request_payload": { + "globExpression": "/foo/*" + }, "response": { - "components": [{ "globExpression": "/foo/" }, { "globExpression": "*" }] + "components": [ + { + "globExpression": "/foo/" + }, + { + "globExpression": "*" + } + ] }, "test_class": "ApiExplainGlobExpressionHandlerTest_http_v2", "url": "/api/v2/clients/C.1000000000000000/glob-expressions:explain" @@ -384,7 +492,9 @@ { "api_method": "ForceRunCronJob", "method": "POST", - "response": { "status": "OK" }, + "response": { + "status": "OK" + }, "test_class": "ApiForceRunCronJobRegressionTest_http_v2", "url": "/api/v2/cron-jobs/FileFinder/actions/force-run" } @@ -394,13 +504,17 @@ "api_method": "GetClientApproval", "method": "GET", "response": { - "approvers": ["api_test_user"], + "approvers": [ + "api_test_user" + ], "emailMessageId": "", "expirationTimeUs": "2419244000000", "id": "approval:111111", "isValid": false, "isValidMessage": "Need at least 1 additional approver for access.", - "notifiedUsers": ["approver"], + "notifiedUsers": [ + "approver" + ], "reason": "foo", "requestor": "api_test_user", "subject": { @@ -409,10 +523,12 @@ "buildTime": "1980-01-01", "clientName": "GRR Monitor", "clientVersion": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "clientId": "C.1000000000000000", - "fleetspeakEnabled": false, "hardwareInfo": { "biosVersion": "Bios-Version-0", "systemManufacturer": "System-Manufacturer-0" @@ -420,7 +536,10 @@ "interfaces": [ { "addresses": [ - { "addressType": "INET", "packedBytes": "wKgAAA==" }, + { + "addressType": "INET", + "packedBytes": "wKgAAA==" + }, { "addressType": "INET6", "packedBytes": "IAGrzQAAAAAAAAAAAAAAAA==" @@ -428,13 +547,26 @@ ], "ifname": "if0" }, - { "ifname": "if1", "macAddress": "qrvM3e4A" }, - { "ifname": "if2", "macAddress": "u8zd7v8A" } + { + "ifname": "if1", + "macAddress": "qrvM3e4A" + }, + { + "ifname": "if2", + "macAddress": "u8zd7v8A" + } ], "knowledgeBase": { "fqdn": "Host-0.example.com", "os": "Linux", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] }, "lastSeenAt": "42000000", "osInfo": { @@ -445,7 +577,14 @@ "version": "buster/sid" }, "urn": "aff4:/C.1000000000000000", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] } }, "test_class": "ApiGetClientApprovalHandlerRegressionTest_http_v2", @@ -455,12 +594,17 @@ "api_method": "GetClientApproval", "method": "GET", "response": { - "approvers": ["api_test_user", "approver"], + "approvers": [ + "api_test_user", + "approver" + ], "emailMessageId": "", "expirationTimeUs": "2419245000000", "id": "approval:222222", "isValid": true, - "notifiedUsers": ["approver"], + "notifiedUsers": [ + "approver" + ], "reason": "bar", "requestor": "api_test_user", "subject": { @@ -469,10 +613,12 @@ "buildTime": "1980-01-01", "clientName": "GRR Monitor", "clientVersion": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "clientId": "C.1000000000000001", - "fleetspeakEnabled": false, "hardwareInfo": { "biosVersion": "Bios-Version-1", "systemManufacturer": "System-Manufacturer-1" @@ -480,7 +626,10 @@ "interfaces": [ { "addresses": [ - { "addressType": "INET", "packedBytes": "wKgAAQ==" }, + { + "addressType": "INET", + "packedBytes": "wKgAAQ==" + }, { "addressType": "INET6", "packedBytes": "IAGrzQAAAAAAAAAAAAAAAQ==" @@ -488,13 +637,26 @@ ], "ifname": "if0" }, - { "ifname": "if1", "macAddress": "qrvM3e4B" }, - { "ifname": "if2", "macAddress": "u8zd7v8B" } + { + "ifname": "if1", + "macAddress": "qrvM3e4B" + }, + { + "ifname": "if2", + "macAddress": "u8zd7v8B" + } ], "knowledgeBase": { "fqdn": "Host-1.example.com", "os": "Linux", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] }, "lastSeenAt": "42000000", "osInfo": { @@ -505,7 +667,14 @@ "version": "buster/sid" }, "urn": "aff4:/C.1000000000000001", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] } }, "test_class": "ApiGetClientApprovalHandlerRegressionTest_http_v2", @@ -522,10 +691,12 @@ "buildTime": "1980-01-01", "clientName": "GRR Monitor", "clientVersion": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "clientId": "C.1000000000000000", - "fleetspeakEnabled": false, "hardwareInfo": { "biosVersion": "Bios-Version-0", "systemManufacturer": "System-Manufacturer-0" @@ -533,7 +704,10 @@ "interfaces": [ { "addresses": [ - { "addressType": "INET", "packedBytes": "wKgAAA==" }, + { + "addressType": "INET", + "packedBytes": "wKgAAA==" + }, { "addressType": "INET6", "packedBytes": "IAGrzQAAAAAAAAAAAAAAAA==" @@ -541,13 +715,26 @@ ], "ifname": "if0" }, - { "ifname": "if1", "macAddress": "qrvM3e4A" }, - { "ifname": "if2", "macAddress": "u8zd7v8A" } + { + "ifname": "if1", + "macAddress": "qrvM3e4A" + }, + { + "ifname": "if2", + "macAddress": "u8zd7v8A" + } ], "knowledgeBase": { "fqdn": "Host-0.example.com", "os": "Linux", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] }, "lastSeenAt": "42000000", "memorySize": "4294967296", @@ -559,7 +746,14 @@ "version": "buster/sid" }, "urn": "aff4:/C.1000000000000000", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] }, "test_class": "ApiGetClientHandlerRegressionTest_http_v2", "url": "/api/v2/clients/C.1000000000000000" @@ -571,8 +765,14 @@ "method": "GET", "response": { "dataPoints": [ - { "timestamp": "10000000", "value": 10.0 }, - { "timestamp": "20000000", "value": 11.0 } + { + "timestamp": "10000000", + "value": 10.0 + }, + { + "timestamp": "20000000", + "value": 11.0 + } ] }, "test_class": "ApiGetClientLoadStatsHandlerRegressionTest_http_v2", @@ -583,8 +783,14 @@ "method": "GET", "response": { "dataPoints": [ - { "timestamp": "10000000", "value": 10.0 }, - { "timestamp": "20000000", "value": 12.0 } + { + "timestamp": "10000000", + "value": 10.0 + }, + { + "timestamp": "20000000", + "value": 12.0 + } ] }, "test_class": "ApiGetClientLoadStatsHandlerRegressionTest_http_v2", @@ -603,7 +809,10 @@ "buildTime": "1980-01-01", "clientName": "GRR Monitor", "clientVersion": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "clientId": "C.1000000000000000", "hardwareInfo": { @@ -613,7 +822,10 @@ "interfaces": [ { "addresses": [ - { "addressType": "INET", "packedBytes": "wKgAAA==" }, + { + "addressType": "INET", + "packedBytes": "wKgAAA==" + }, { "addressType": "INET6", "packedBytes": "IAGrzQAAAAAAAAAAAAAAAA==" @@ -621,13 +833,26 @@ ], "ifname": "if0" }, - { "ifname": "if1", "macAddress": "qrvM3e4A" }, - { "ifname": "if2", "macAddress": "u8zd7v8A" } + { + "ifname": "if1", + "macAddress": "qrvM3e4A" + }, + { + "ifname": "if2", + "macAddress": "u8zd7v8A" + } ], "knowledgeBase": { "fqdn": "Host-0.example.com", "os": "Linux", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] }, "memorySize": "4294967296", "osInfo": { @@ -638,7 +863,14 @@ "version": "buster/sid" }, "urn": "aff4:/C.1000000000000000", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] }, { "age": "45000000", @@ -646,7 +878,10 @@ "buildTime": "1980-01-01", "clientName": "GRR Monitor", "clientVersion": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "clientId": "C.1000000000000000", "hardwareInfo": { @@ -656,7 +891,10 @@ "interfaces": [ { "addresses": [ - { "addressType": "INET", "packedBytes": "wKgAAA==" }, + { + "addressType": "INET", + "packedBytes": "wKgAAA==" + }, { "addressType": "INET6", "packedBytes": "IAGrzQAAAAAAAAAAAAAAAA==" @@ -664,13 +902,26 @@ ], "ifname": "if0" }, - { "ifname": "if1", "macAddress": "qrvM3e4A" }, - { "ifname": "if2", "macAddress": "u8zd7v8A" } + { + "ifname": "if1", + "macAddress": "qrvM3e4A" + }, + { + "ifname": "if2", + "macAddress": "u8zd7v8A" + } ], "knowledgeBase": { "fqdn": "some-other-hostname.org", "os": "Linux", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] }, "memorySize": "4294967296", "osInfo": { @@ -681,7 +932,14 @@ "version": "buster/sid" }, "urn": "aff4:/C.1000000000000000", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] } ] }, @@ -699,7 +957,10 @@ "buildTime": "1980-01-01", "clientName": "GRR Monitor", "clientVersion": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "clientId": "C.1000000000000000", "hardwareInfo": { @@ -709,7 +970,10 @@ "interfaces": [ { "addresses": [ - { "addressType": "INET", "packedBytes": "wKgAAA==" }, + { + "addressType": "INET", + "packedBytes": "wKgAAA==" + }, { "addressType": "INET6", "packedBytes": "IAGrzQAAAAAAAAAAAAAAAA==" @@ -717,13 +981,26 @@ ], "ifname": "if0" }, - { "ifname": "if1", "macAddress": "qrvM3e4A" }, - { "ifname": "if2", "macAddress": "u8zd7v8A" } + { + "ifname": "if1", + "macAddress": "qrvM3e4A" + }, + { + "ifname": "if2", + "macAddress": "u8zd7v8A" + } ], "knowledgeBase": { "fqdn": "Host-0.example.com", "os": "Linux", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] }, "memorySize": "4294967296", "osInfo": { @@ -734,7 +1011,14 @@ "version": "buster/sid" }, "urn": "aff4:/C.1000000000000000", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] } ] }, @@ -752,7 +1036,10 @@ "buildTime": "1980-01-01", "clientName": "GRR Monitor", "clientVersion": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "clientId": "C.1000000000000000", "hardwareInfo": { @@ -762,7 +1049,10 @@ "interfaces": [ { "addresses": [ - { "addressType": "INET", "packedBytes": "wKgAAA==" }, + { + "addressType": "INET", + "packedBytes": "wKgAAA==" + }, { "addressType": "INET6", "packedBytes": "IAGrzQAAAAAAAAAAAAAAAA==" @@ -770,13 +1060,26 @@ ], "ifname": "if0" }, - { "ifname": "if1", "macAddress": "qrvM3e4A" }, - { "ifname": "if2", "macAddress": "u8zd7v8A" } + { + "ifname": "if1", + "macAddress": "qrvM3e4A" + }, + { + "ifname": "if2", + "macAddress": "u8zd7v8A" + } ], "knowledgeBase": { "fqdn": "some-other-hostname.org", "os": "Linux", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] }, "memorySize": "4294967296", "osInfo": { @@ -787,7 +1090,14 @@ "version": "buster/sid" }, "urn": "aff4:/C.1000000000000000", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] } ] }, @@ -800,12 +1110,16 @@ "api_method": "GetCronJobApproval", "method": "GET", "response": { - "approvers": ["api_test_user"], + "approvers": [ + "api_test_user" + ], "emailMessageId": "", "id": "approval:111111", "isValid": false, "isValidMessage": "Need at least 1 additional approver for access.", - "notifiedUsers": ["approver"], + "notifiedUsers": [ + "approver" + ], "reason": "foo", "requestor": "api_test_user", "subject": { @@ -838,11 +1152,16 @@ "api_method": "GetCronJobApproval", "method": "GET", "response": { - "approvers": ["api_test_user", "approver"], + "approvers": [ + "api_test_user", + "approver" + ], "emailMessageId": "", "id": "approval:222222", "isValid": true, - "notifiedUsers": ["approver"], + "notifiedUsers": [ + "approver" + ], "reason": "bar", "requestor": "api_test_user", "subject": { @@ -902,21 +1221,30 @@ { "api_method": "GetFileText", "method": "GET", - "response": { "content": "Goodbye World", "totalSize": "13" }, + "response": { + "content": "Goodbye World", + "totalSize": "13" + }, "test_class": "ApiGetFileTextHandlerRegressionTest_http_v2", "url": "/api/v2/clients/C.1000000000000000/vfs-text/fs/os/c/Downloads/a.txt" }, { "api_method": "GetFileText", "method": "GET", - "response": { "content": "dby", "totalSize": "13" }, + "response": { + "content": "dby", + "totalSize": "13" + }, "test_class": "ApiGetFileTextHandlerRegressionTest_http_v2", "url": "/api/v2/clients/C.1000000000000000/vfs-text/fs/os/c/Downloads/a.txt?length=3&offset=3" }, { "api_method": "GetFileText", "method": "GET", - "response": { "content": "Hello World", "totalSize": "11" }, + "response": { + "content": "Hello World", + "totalSize": "11" + }, "test_class": "ApiGetFileTextHandlerRegressionTest_http_v2", "url": "/api/v2/clients/C.1000000000000000/vfs-text/fs/os/c/Downloads/a.txt?timestamp=86400000042" } @@ -925,7 +1253,12 @@ { "api_method": "GetFileVersionTimes", "method": "GET", - "response": { "times": ["172800000042", "86400000042"] }, + "response": { + "times": [ + "172800000042", + "86400000042" + ] + }, "test_class": "ApiGetFileVersionTimesHandlerRegressionTest_http_v2", "url": "/api/v2/clients/C.1000000000000000/vfs-version-times/fs/os/c/Downloads/a.txt" } @@ -940,7 +1273,9 @@ }, "clientId": "C.1000000000000000", "context": { - "clientResources": { "cpuUsage": {} }, + "clientResources": { + "cpuUsage": {} + }, "createTime": "42000000", "creator": "api_test_user", "currentState": "Start", @@ -957,10 +1292,14 @@ "progress": { "@type": "type.googleapis.com/grr.DefaultFlowProgress" }, - "resultMetadata": { "isMetadataSet": true }, + "resultMetadata": { + "isMetadataSet": true + }, "runnerArgs": { "clientId": "aff4:/C.1000000000000000", + "cpuLimit": "60", "flowName": "Interrogate", + "networkBytesLimit": "8192", "notifyToUser": true }, "startedAt": "42000000", @@ -981,11 +1320,19 @@ "value": { "@type": "type.googleapis.com/grr.ClientSnapshot", "clientId": "C.1000000000000000", - "metadata": { "sourceFlowId": "F:ABCDEF12" } + "metadata": { + "sourceFlowId": "F:ABCDEF12" + } } }, - { "invalid": true, "key": "fqdn" }, - { "invalid": true, "key": "os" } + { + "invalid": true, + "key": "fqdn" + }, + { + "invalid": true, + "key": "os" + } ] }, "urn": "aff4:/C.1000000000000000/flows/F:ABCDEF12" @@ -1002,7 +1349,9 @@ }, "clientId": "C.1000000000000000", "context": { - "clientResources": { "cpuUsage": {} }, + "clientResources": { + "cpuUsage": {} + }, "createTime": "42000000", "creator": "api_test_user", "currentState": "Start", @@ -1021,10 +1370,14 @@ "progress": { "@type": "type.googleapis.com/grr.DefaultFlowProgress" }, - "resultMetadata": { "isMetadataSet": true }, + "resultMetadata": { + "isMetadataSet": true + }, "runnerArgs": { "clientId": "aff4:/C.1000000000000000", + "cpuLimit": "60", "flowName": "Interrogate", + "networkBytesLimit": "8192", "notifyToUser": true }, "startedAt": "42000000", @@ -1045,11 +1398,19 @@ "value": { "@type": "type.googleapis.com/grr.ClientSnapshot", "clientId": "C.1000000000000000", - "metadata": { "sourceFlowId": "F:ABCDEF13" } + "metadata": { + "sourceFlowId": "F:ABCDEF13" + } } }, - { "invalid": true, "key": "fqdn" }, - { "invalid": true, "key": "os" } + { + "invalid": true, + "key": "fqdn" + }, + { + "invalid": true, + "key": "os" + } ] }, "urn": "aff4:/C.1000000000000000/flows/F:ABCDEF13" @@ -1118,13 +1479,17 @@ "api_method": "GetHuntApproval", "method": "GET", "response": { - "approvers": ["api_test_user"], + "approvers": [ + "api_test_user" + ], "emailMessageId": "", "expirationTimeUs": "2419244000000", "id": "approval:111111", "isValid": false, "isValidMessage": "Need at least 1 additional approver for access.", - "notifiedUsers": ["approver"], + "notifiedUsers": [ + "approver" + ], "reason": "foo", "requestor": "api_test_user", "subject": { @@ -1134,7 +1499,10 @@ "clientRuleSet": { "rules": [ { - "regex": { "attributeRegex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attributeRegex": "GRR", + "field": "CLIENT_NAME" + }, "ruleType": "REGEX" } ] @@ -1150,7 +1518,10 @@ "failedClientsCount": "0", "flowArgs": { "@type": "type.googleapis.com/grr.GetFileArgs", - "pathspec": { "path": "/tmp/evil.txt", "pathtype": "OS" } + "pathspec": { + "path": "/tmp/evil.txt", + "pathtype": "OS" + } }, "flowName": "GetFile", "huntId": "H:123456", @@ -1162,7 +1533,10 @@ "clientRuleSet": { "rules": [ { - "regex": { "attributeRegex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attributeRegex": "GRR", + "field": "CLIENT_NAME" + }, "ruleType": "REGEX" } ] @@ -1180,6 +1554,7 @@ "resultsCount": "0", "state": "PAUSED", "stateComment": "", + "stateReason": "UNKNOWN", "totalCpuUsage": 0.0, "totalNetUsage": "0", "urn": "aff4:/hunts/H:123456" @@ -1192,12 +1567,17 @@ "api_method": "GetHuntApproval", "method": "GET", "response": { - "approvers": ["api_test_user", "approver"], + "approvers": [ + "api_test_user", + "approver" + ], "emailMessageId": "", "expirationTimeUs": "2419245000000", "id": "approval:222222", "isValid": true, - "notifiedUsers": ["approver"], + "notifiedUsers": [ + "approver" + ], "reason": "bar", "requestor": "api_test_user", "subject": { @@ -1207,7 +1587,10 @@ "clientRuleSet": { "rules": [ { - "regex": { "attributeRegex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attributeRegex": "GRR", + "field": "CLIENT_NAME" + }, "ruleType": "REGEX" } ] @@ -1223,7 +1606,10 @@ "failedClientsCount": "0", "flowArgs": { "@type": "type.googleapis.com/grr.GetFileArgs", - "pathspec": { "path": "/tmp/evil.txt", "pathtype": "OS" } + "pathspec": { + "path": "/tmp/evil.txt", + "pathtype": "OS" + } }, "flowName": "GetFile", "huntId": "H:567890", @@ -1235,7 +1621,10 @@ "clientRuleSet": { "rules": [ { - "regex": { "attributeRegex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attributeRegex": "GRR", + "field": "CLIENT_NAME" + }, "ruleType": "REGEX" } ] @@ -1253,6 +1642,7 @@ "resultsCount": "0", "state": "PAUSED", "stateComment": "", + "stateReason": "UNKNOWN", "totalCpuUsage": 0.0, "totalNetUsage": "0", "urn": "aff4:/hunts/H:567890" @@ -1265,7 +1655,9 @@ "api_method": "GetHuntApproval", "method": "GET", "response": { - "approvers": ["api_test_user"], + "approvers": [ + "api_test_user" + ], "copiedFromHunt": { "allClientsCount": "0", "clientLimit": "100", @@ -1273,7 +1665,10 @@ "clientRuleSet": { "rules": [ { - "regex": { "attributeRegex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attributeRegex": "GRR", + "field": "CLIENT_NAME" + }, "ruleType": "REGEX" } ] @@ -1289,7 +1684,10 @@ "failedClientsCount": "0", "flowArgs": { "@type": "type.googleapis.com/grr.GetFileArgs", - "pathspec": { "path": "/tmp/evil.txt", "pathtype": "OS" } + "pathspec": { + "path": "/tmp/evil.txt", + "pathtype": "OS" + } }, "flowName": "GetFile", "huntId": "H:556677", @@ -1301,7 +1699,10 @@ "clientRuleSet": { "rules": [ { - "regex": { "attributeRegex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attributeRegex": "GRR", + "field": "CLIENT_NAME" + }, "ruleType": "REGEX" } ] @@ -1319,6 +1720,7 @@ "resultsCount": "0", "state": "PAUSED", "stateComment": "", + "stateReason": "UNKNOWN", "totalCpuUsage": 0.0, "totalNetUsage": "0", "urn": "aff4:/hunts/H:556677" @@ -1328,7 +1730,9 @@ "id": "approval:333333", "isValid": false, "isValidMessage": "Need at least 1 additional approver for access.", - "notifiedUsers": ["approver"], + "notifiedUsers": [ + "approver" + ], "reason": "foo", "requestor": "api_test_user", "subject": { @@ -1338,7 +1742,10 @@ "clientRuleSet": { "rules": [ { - "regex": { "attributeRegex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attributeRegex": "GRR", + "field": "CLIENT_NAME" + }, "ruleType": "REGEX" } ] @@ -1354,7 +1761,10 @@ "failedClientsCount": "0", "flowArgs": { "@type": "type.googleapis.com/grr.GetFileArgs", - "pathspec": { "path": "/tmp/evil.txt", "pathtype": "OS" } + "pathspec": { + "path": "/tmp/evil.txt", + "pathtype": "OS" + } }, "flowName": "GetFile", "huntId": "H:DDEEFF", @@ -1366,7 +1776,10 @@ "clientRuleSet": { "rules": [ { - "regex": { "attributeRegex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attributeRegex": "GRR", + "field": "CLIENT_NAME" + }, "ruleType": "REGEX" } ] @@ -1376,7 +1789,9 @@ "expiryTime": "1209600", "huntName": "GenericHunt", "originalObject": { - "huntReference": { "huntId": "H:556677" }, + "huntReference": { + "huntId": "H:556677" + }, "objectType": "HUNT_REFERENCE" } }, @@ -1384,13 +1799,16 @@ "isRobot": false, "name": "GenericHunt", "originalObject": { - "huntReference": { "huntId": "H:556677" }, + "huntReference": { + "huntId": "H:556677" + }, "objectType": "HUNT_REFERENCE" }, "remainingClientsCount": "0", "resultsCount": "0", "state": "PAUSED", "stateComment": "", + "stateReason": "UNKNOWN", "totalCpuUsage": 0.0, "totalNetUsage": "0", "urn": "aff4:/hunts/H:DDEEFF" @@ -1403,7 +1821,9 @@ "api_method": "GetHuntApproval", "method": "GET", "response": { - "approvers": ["api_test_user"], + "approvers": [ + "api_test_user" + ], "copiedFromFlow": { "args": { "@type": "type.googleapis.com/grr.InterrogateArgs" @@ -1428,7 +1848,9 @@ "id": "approval:444444", "isValid": false, "isValidMessage": "Need at least 1 additional approver for access.", - "notifiedUsers": ["approver"], + "notifiedUsers": [ + "approver" + ], "reason": "foo", "requestor": "api_test_user", "subject": { @@ -1438,7 +1860,10 @@ "clientRuleSet": { "rules": [ { - "regex": { "attributeRegex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attributeRegex": "GRR", + "field": "CLIENT_NAME" + }, "ruleType": "REGEX" } ] @@ -1454,7 +1879,10 @@ "failedClientsCount": "0", "flowArgs": { "@type": "type.googleapis.com/grr.GetFileArgs", - "pathspec": { "path": "/tmp/evil.txt", "pathtype": "OS" } + "pathspec": { + "path": "/tmp/evil.txt", + "pathtype": "OS" + } }, "flowName": "GetFile", "huntId": "H:667788", @@ -1466,7 +1894,10 @@ "clientRuleSet": { "rules": [ { - "regex": { "attributeRegex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attributeRegex": "GRR", + "field": "CLIENT_NAME" + }, "ruleType": "REGEX" } ] @@ -1497,6 +1928,7 @@ "resultsCount": "0", "state": "PAUSED", "stateComment": "", + "stateReason": "UNKNOWN", "totalCpuUsage": 0.0, "totalNetUsage": "0", "urn": "aff4:/hunts/H:667788" @@ -1512,30 +1944,96 @@ "method": "GET", "response": { "completePoints": [ - { "xValue": 44.0, "yValue": 0.0 }, - { "xValue": 45.0, "yValue": 1.0 }, - { "xValue": 55.0, "yValue": 2.0 }, - { "xValue": 65.0, "yValue": 3.0 }, - { "xValue": 75.0, "yValue": 4.0 }, - { "xValue": 85.0, "yValue": 5.0 }, - { "xValue": 95.0, "yValue": 6.0 }, - { "xValue": 105.0, "yValue": 7.0 }, - { "xValue": 115.0, "yValue": 8.0 }, - { "xValue": 125.0, "yValue": 9.0 }, - { "xValue": 135.0, "yValue": 10.0 } + { + "xValue": 44.0, + "yValue": 0.0 + }, + { + "xValue": 45.0, + "yValue": 1.0 + }, + { + "xValue": 55.0, + "yValue": 2.0 + }, + { + "xValue": 65.0, + "yValue": 3.0 + }, + { + "xValue": 75.0, + "yValue": 4.0 + }, + { + "xValue": 85.0, + "yValue": 5.0 + }, + { + "xValue": 95.0, + "yValue": 6.0 + }, + { + "xValue": 105.0, + "yValue": 7.0 + }, + { + "xValue": 115.0, + "yValue": 8.0 + }, + { + "xValue": 125.0, + "yValue": 9.0 + }, + { + "xValue": 135.0, + "yValue": 10.0 + } ], "startPoints": [ - { "xValue": 44.0, "yValue": 0.0 }, - { "xValue": 45.0, "yValue": 1.0 }, - { "xValue": 55.0, "yValue": 2.0 }, - { "xValue": 65.0, "yValue": 3.0 }, - { "xValue": 75.0, "yValue": 4.0 }, - { "xValue": 85.0, "yValue": 5.0 }, - { "xValue": 95.0, "yValue": 6.0 }, - { "xValue": 105.0, "yValue": 7.0 }, - { "xValue": 115.0, "yValue": 8.0 }, - { "xValue": 125.0, "yValue": 9.0 }, - { "xValue": 135.0, "yValue": 10.0 } + { + "xValue": 44.0, + "yValue": 0.0 + }, + { + "xValue": 45.0, + "yValue": 1.0 + }, + { + "xValue": 55.0, + "yValue": 2.0 + }, + { + "xValue": 65.0, + "yValue": 3.0 + }, + { + "xValue": 75.0, + "yValue": 4.0 + }, + { + "xValue": 85.0, + "yValue": 5.0 + }, + { + "xValue": 95.0, + "yValue": 6.0 + }, + { + "xValue": 105.0, + "yValue": 7.0 + }, + { + "xValue": 115.0, + "yValue": 8.0 + }, + { + "xValue": 125.0, + "yValue": 9.0 + }, + { + "xValue": 135.0, + "yValue": 10.0 + } ] }, "test_class": "ApiGetHuntClientCompletionStatsHandlerRegressionTest_http_v2", @@ -1546,16 +2044,40 @@ "method": "GET", "response": { "completePoints": [ - { "xValue": 66.75, "yValue": 3.0 }, - { "xValue": 89.5, "yValue": 5.0 }, - { "xValue": 112.25, "yValue": 7.0 }, - { "xValue": 135.0, "yValue": 10.0 } + { + "xValue": 66.75, + "yValue": 3.0 + }, + { + "xValue": 89.5, + "yValue": 5.0 + }, + { + "xValue": 112.25, + "yValue": 7.0 + }, + { + "xValue": 135.0, + "yValue": 10.0 + } ], "startPoints": [ - { "xValue": 66.75, "yValue": 3.0 }, - { "xValue": 89.5, "yValue": 5.0 }, - { "xValue": 112.25, "yValue": 7.0 }, - { "xValue": 135.0, "yValue": 10.0 } + { + "xValue": 66.75, + "yValue": 3.0 + }, + { + "xValue": 89.5, + "yValue": 5.0 + }, + { + "xValue": 112.25, + "yValue": 7.0 + }, + { + "xValue": 135.0, + "yValue": 10.0 + } ] }, "test_class": "ApiGetHuntClientCompletionStatsHandlerRegressionTest_http_v2", @@ -1566,30 +2088,96 @@ "method": "GET", "response": { "completePoints": [ - { "xValue": 44.0, "yValue": 0.0 }, - { "xValue": 45.0, "yValue": 1.0 }, - { "xValue": 55.0, "yValue": 2.0 }, - { "xValue": 65.0, "yValue": 3.0 }, - { "xValue": 75.0, "yValue": 4.0 }, - { "xValue": 85.0, "yValue": 5.0 }, - { "xValue": 95.0, "yValue": 6.0 }, - { "xValue": 105.0, "yValue": 7.0 }, - { "xValue": 115.0, "yValue": 8.0 }, - { "xValue": 125.0, "yValue": 9.0 }, - { "xValue": 135.0, "yValue": 10.0 } + { + "xValue": 44.0, + "yValue": 0.0 + }, + { + "xValue": 45.0, + "yValue": 1.0 + }, + { + "xValue": 55.0, + "yValue": 2.0 + }, + { + "xValue": 65.0, + "yValue": 3.0 + }, + { + "xValue": 75.0, + "yValue": 4.0 + }, + { + "xValue": 85.0, + "yValue": 5.0 + }, + { + "xValue": 95.0, + "yValue": 6.0 + }, + { + "xValue": 105.0, + "yValue": 7.0 + }, + { + "xValue": 115.0, + "yValue": 8.0 + }, + { + "xValue": 125.0, + "yValue": 9.0 + }, + { + "xValue": 135.0, + "yValue": 10.0 + } ], "startPoints": [ - { "xValue": 44.0, "yValue": 0.0 }, - { "xValue": 45.0, "yValue": 1.0 }, - { "xValue": 55.0, "yValue": 2.0 }, - { "xValue": 65.0, "yValue": 3.0 }, - { "xValue": 75.0, "yValue": 4.0 }, - { "xValue": 85.0, "yValue": 5.0 }, - { "xValue": 95.0, "yValue": 6.0 }, - { "xValue": 105.0, "yValue": 7.0 }, - { "xValue": 115.0, "yValue": 8.0 }, - { "xValue": 125.0, "yValue": 9.0 }, - { "xValue": 135.0, "yValue": 10.0 } + { + "xValue": 44.0, + "yValue": 0.0 + }, + { + "xValue": 45.0, + "yValue": 1.0 + }, + { + "xValue": 55.0, + "yValue": 2.0 + }, + { + "xValue": 65.0, + "yValue": 3.0 + }, + { + "xValue": 75.0, + "yValue": 4.0 + }, + { + "xValue": 85.0, + "yValue": 5.0 + }, + { + "xValue": 95.0, + "yValue": 6.0 + }, + { + "xValue": 105.0, + "yValue": 7.0 + }, + { + "xValue": 115.0, + "yValue": 8.0 + }, + { + "xValue": 125.0, + "yValue": 9.0 + }, + { + "xValue": 135.0, + "yValue": 10.0 + } ] }, "test_class": "ApiGetHuntClientCompletionStatsHandlerRegressionTest_http_v2", @@ -1607,7 +2195,10 @@ "clientRuleSet": { "rules": [ { - "regex": { "attributeRegex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attributeRegex": "GRR", + "field": "CLIENT_NAME" + }, "ruleType": "REGEX" } ] @@ -1623,7 +2214,10 @@ "failedClientsCount": "0", "flowArgs": { "@type": "type.googleapis.com/grr.GetFileArgs", - "pathspec": { "path": "/tmp/evil.txt", "pathtype": "OS" } + "pathspec": { + "path": "/tmp/evil.txt", + "pathtype": "OS" + } }, "flowName": "GetFile", "huntId": "H:123456", @@ -1635,7 +2229,10 @@ "clientRuleSet": { "rules": [ { - "regex": { "attributeRegex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attributeRegex": "GRR", + "field": "CLIENT_NAME" + }, "ruleType": "REGEX" } ] @@ -1666,6 +2263,7 @@ "resultsCount": "0", "state": "PAUSED", "stateComment": "", + "stateReason": "UNKNOWN", "totalCpuUsage": 0.0, "totalNetUsage": "0", "urn": "aff4:/hunts/H:123456" @@ -1683,7 +2281,10 @@ "clientRuleSet": { "rules": [ { - "regex": { "attributeRegex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attributeRegex": "GRR", + "field": "CLIENT_NAME" + }, "ruleType": "REGEX" } ] @@ -1699,7 +2300,10 @@ "failedClientsCount": "0", "flowArgs": { "@type": "type.googleapis.com/grr.GetFileArgs", - "pathspec": { "path": "/tmp/evil.txt", "pathtype": "OS" } + "pathspec": { + "path": "/tmp/evil.txt", + "pathtype": "OS" + } }, "flowName": "GetFile", "huntId": "H:123456", @@ -1711,7 +2315,10 @@ "clientRuleSet": { "rules": [ { - "regex": { "attributeRegex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attributeRegex": "GRR", + "field": "CLIENT_NAME" + }, "ruleType": "REGEX" } ] @@ -1721,7 +2328,9 @@ "expiryTime": "1209600", "huntName": "GenericHunt", "originalObject": { - "huntReference": { "huntId": "H:332211" }, + "huntReference": { + "huntId": "H:332211" + }, "objectType": "HUNT_REFERENCE" } }, @@ -1729,13 +2338,16 @@ "isRobot": false, "name": "GenericHunt", "originalObject": { - "huntReference": { "huntId": "H:332211" }, + "huntReference": { + "huntId": "H:332211" + }, "objectType": "HUNT_REFERENCE" }, "remainingClientsCount": "0", "resultsCount": "0", "state": "PAUSED", "stateComment": "", + "stateReason": "UNKNOWN", "totalCpuUsage": 0.0, "totalNetUsage": "0", "urn": "aff4:/hunts/H:123456" @@ -1753,7 +2365,10 @@ "clientRuleSet": { "rules": [ { - "regex": { "attributeRegex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attributeRegex": "GRR", + "field": "CLIENT_NAME" + }, "ruleType": "REGEX" } ] @@ -1769,7 +2384,10 @@ "failedClientsCount": "0", "flowArgs": { "@type": "type.googleapis.com/grr.GetFileArgs", - "pathspec": { "path": "/tmp/evil.txt", "pathtype": "OS" } + "pathspec": { + "path": "/tmp/evil.txt", + "pathtype": "OS" + } }, "flowName": "GetFile", "huntId": "H:123456", @@ -1781,7 +2399,10 @@ "clientRuleSet": { "rules": [ { - "regex": { "attributeRegex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attributeRegex": "GRR", + "field": "CLIENT_NAME" + }, "ruleType": "REGEX" } ] @@ -1799,6 +2420,7 @@ "resultsCount": "0", "state": "PAUSED", "stateComment": "", + "stateReason": "UNKNOWN", "totalCpuUsage": 0.0, "totalNetUsage": "0", "urn": "aff4:/hunts/H:123456" @@ -1827,24 +2449,61 @@ "networkBytesSentStats": { "histogram": { "bins": [ - { "num": "1", "rangeMaxValue": 16.0 }, - { "rangeMaxValue": 32.0 }, - { "rangeMaxValue": 64.0 }, - { "rangeMaxValue": 128.0 }, - { "rangeMaxValue": 256.0 }, - { "rangeMaxValue": 512.0 }, - { "rangeMaxValue": 1024.0 }, - { "rangeMaxValue": 2048.0 }, - { "rangeMaxValue": 4096.0 }, - { "rangeMaxValue": 8192.0 }, - { "rangeMaxValue": 16384.0 }, - { "rangeMaxValue": 32768.0 }, - { "rangeMaxValue": 65536.0 }, - { "rangeMaxValue": 131072.0 }, - { "rangeMaxValue": 262144.0 }, - { "rangeMaxValue": 524288.0 }, - { "rangeMaxValue": 1048576.0 }, - { "rangeMaxValue": 2097152.0 } + { + "num": "1", + "rangeMaxValue": 16.0 + }, + { + "rangeMaxValue": 32.0 + }, + { + "rangeMaxValue": 64.0 + }, + { + "rangeMaxValue": 128.0 + }, + { + "rangeMaxValue": 256.0 + }, + { + "rangeMaxValue": 512.0 + }, + { + "rangeMaxValue": 1024.0 + }, + { + "rangeMaxValue": 2048.0 + }, + { + "rangeMaxValue": 4096.0 + }, + { + "rangeMaxValue": 8192.0 + }, + { + "rangeMaxValue": 16384.0 + }, + { + "rangeMaxValue": 32768.0 + }, + { + "rangeMaxValue": 65536.0 + }, + { + "rangeMaxValue": 131072.0 + }, + { + "rangeMaxValue": 262144.0 + }, + { + "rangeMaxValue": 524288.0 + }, + { + "rangeMaxValue": 1048576.0 + }, + { + "rangeMaxValue": 2097152.0 + } ] }, "num": "1", @@ -1854,26 +2513,67 @@ "systemCpuStats": { "histogram": { "bins": [ - { "rangeMaxValue": 0.1 }, - { "rangeMaxValue": 0.2 }, - { "rangeMaxValue": 0.30000001 }, - { "rangeMaxValue": 0.40000001 }, - { "rangeMaxValue": 0.5 }, - { "rangeMaxValue": 0.75 }, - { "rangeMaxValue": 1.0 }, - { "rangeMaxValue": 1.5 }, - { "rangeMaxValue": 2.0 }, - { "num": "1", "rangeMaxValue": 2.5 }, - { "rangeMaxValue": 3.0 }, - { "rangeMaxValue": 4.0 }, - { "rangeMaxValue": 5.0 }, - { "rangeMaxValue": 6.0 }, - { "rangeMaxValue": 7.0 }, - { "rangeMaxValue": 8.0 }, - { "rangeMaxValue": 9.0 }, - { "rangeMaxValue": 10.0 }, - { "rangeMaxValue": 15.0 }, - { "rangeMaxValue": 20.0 } + { + "rangeMaxValue": 0.1 + }, + { + "rangeMaxValue": 0.2 + }, + { + "rangeMaxValue": 0.30000001 + }, + { + "rangeMaxValue": 0.40000001 + }, + { + "rangeMaxValue": 0.5 + }, + { + "rangeMaxValue": 0.75 + }, + { + "rangeMaxValue": 1.0 + }, + { + "rangeMaxValue": 1.5 + }, + { + "rangeMaxValue": 2.0 + }, + { + "num": "1", + "rangeMaxValue": 2.5 + }, + { + "rangeMaxValue": 3.0 + }, + { + "rangeMaxValue": 4.0 + }, + { + "rangeMaxValue": 5.0 + }, + { + "rangeMaxValue": 6.0 + }, + { + "rangeMaxValue": 7.0 + }, + { + "rangeMaxValue": 8.0 + }, + { + "rangeMaxValue": 9.0 + }, + { + "rangeMaxValue": 10.0 + }, + { + "rangeMaxValue": 15.0 + }, + { + "rangeMaxValue": 20.0 + } ] }, "num": "1", @@ -1883,26 +2583,67 @@ "userCpuStats": { "histogram": { "bins": [ - { "rangeMaxValue": 0.1 }, - { "rangeMaxValue": 0.2 }, - { "rangeMaxValue": 0.30000001 }, - { "rangeMaxValue": 0.40000001 }, - { "rangeMaxValue": 0.5 }, - { "rangeMaxValue": 0.75 }, - { "rangeMaxValue": 1.0 }, - { "num": "1", "rangeMaxValue": 1.5 }, - { "rangeMaxValue": 2.0 }, - { "rangeMaxValue": 2.5 }, - { "rangeMaxValue": 3.0 }, - { "rangeMaxValue": 4.0 }, - { "rangeMaxValue": 5.0 }, - { "rangeMaxValue": 6.0 }, - { "rangeMaxValue": 7.0 }, - { "rangeMaxValue": 8.0 }, - { "rangeMaxValue": 9.0 }, - { "rangeMaxValue": 10.0 }, - { "rangeMaxValue": 15.0 }, - { "rangeMaxValue": 20.0 } + { + "rangeMaxValue": 0.1 + }, + { + "rangeMaxValue": 0.2 + }, + { + "rangeMaxValue": 0.30000001 + }, + { + "rangeMaxValue": 0.40000001 + }, + { + "rangeMaxValue": 0.5 + }, + { + "rangeMaxValue": 0.75 + }, + { + "rangeMaxValue": 1.0 + }, + { + "num": "1", + "rangeMaxValue": 1.5 + }, + { + "rangeMaxValue": 2.0 + }, + { + "rangeMaxValue": 2.5 + }, + { + "rangeMaxValue": 3.0 + }, + { + "rangeMaxValue": 4.0 + }, + { + "rangeMaxValue": 5.0 + }, + { + "rangeMaxValue": 6.0 + }, + { + "rangeMaxValue": 7.0 + }, + { + "rangeMaxValue": 8.0 + }, + { + "rangeMaxValue": 9.0 + }, + { + "rangeMaxValue": 10.0 + }, + { + "rangeMaxValue": 15.0 + }, + { + "rangeMaxValue": 20.0 + } ] }, "num": "1", @@ -1912,7 +2653,10 @@ "worstPerformers": [ { "clientId": "aff4:/C.1000000000000000", - "cpuUsage": { "systemCpuTime": 2.0, "userCpuTime": 1.0 }, + "cpuUsage": { + "systemCpuTime": 2.0, + "userCpuTime": 1.0 + }, "networkBytesSent": "3", "sessionId": "" } @@ -1928,9 +2672,9 @@ "api_method": "GetLastClientIPAddress", "method": "GET", "response": { - "info": "Internal IP address.", - "ip": "192.168.100.42", - "status": "INTERNAL" + "info": "No ip information.", + "ip": "", + "status": "UNKNOWN" }, "test_class": "ApiGetLastClientIPAddressHandlerRegressionTest_http_v2", "url": "/api/v2/clients/C.1000000000000000/last-ip" @@ -1960,7 +2704,10 @@ "uploadArtifactActionEnabled": true, "uploadBinaryActionEnabled": true }, - "settings": { "canaryMode": true, "mode": "ADVANCED" }, + "settings": { + "canaryMode": true, + "mode": "ADVANCED" + }, "userType": "USER_TYPE_STANDARD", "username": "api_test_user" }, @@ -1990,7 +2737,10 @@ "uploadArtifactActionEnabled": true, "uploadBinaryActionEnabled": true }, - "settings": { "canaryMode": true, "mode": "ADVANCED" }, + "settings": { + "canaryMode": true, + "mode": "ADVANCED" + }, "userType": "USER_TYPE_ADMIN", "username": "api_test_user" }, @@ -2002,7 +2752,9 @@ { "api_method": "GetPendingUserNotificationsCount", "method": "GET", - "response": { "count": "2" }, + "response": { + "count": "2" + }, "test_class": "ApiGetPendingUserNotificationsCountHandlerRegressionTest_http_v2", "url": "/api/v2/users/me/notifications/pending/count" } @@ -2115,10 +2867,26 @@ }, { "allowedValues": [ - { "doc": "", "name": "RUNNING", "value": "0" }, - { "doc": "", "name": "TERMINATED", "value": "1" }, - { "doc": "", "name": "ERROR", "value": "3" }, - { "doc": "", "name": "CLIENT_CRASHED", "value": "4" } + { + "doc": "", + "name": "RUNNING", + "value": "0" + }, + { + "doc": "", + "name": "TERMINATED", + "value": "1" + }, + { + "doc": "", + "name": "ERROR", + "value": "3" + }, + { + "doc": "", + "name": "CLIENT_CRASHED", + "value": "4" + } ], "default": { "@type": "type.googleapis.com/google.protobuf.Int64Value", @@ -2221,7 +2989,9 @@ "dynamic": false, "friendlyName": "Original flow", "index": 13, - "labels": ["HIDDEN"], + "labels": [ + "HIDDEN" + ], "name": "original_flow", "repeated": false, "type": "ApiFlowReference" @@ -2266,10 +3036,22 @@ { "label": "Bar", "points": [ - { "x": 5.0, "y": 3.0 }, - { "x": 8.0, "y": 4.0 }, - { "x": 13.0, "y": 5.0 }, - { "x": 21.0, "y": 6.0 } + { + "x": 5.0, + "y": 3.0 + }, + { + "x": 8.0, + "y": 4.0 + }, + { + "x": 13.0, + "y": 5.0 + }, + { + "x": 21.0, + "y": 6.0 + } ] } ] @@ -2291,28 +3073,36 @@ { "api_method": "GetVfsFileContentUpdateState", "method": "GET", - "response": { "state": "RUNNING" }, + "response": { + "state": "RUNNING" + }, "test_class": "ApiGetVfsFileContentUpdateStateHandlerRegressionTest_http_v2", "url": "/api/v2/clients/C.1000000000000000/vfs-update/ABCDEF" }, { "api_method": "GetVfsFileContentUpdateState", "method": "GET", - "response": { "state": "FINISHED" }, + "response": { + "state": "FINISHED" + }, "test_class": "ApiGetVfsFileContentUpdateStateHandlerRegressionTest_http_v2", "url": "/api/v2/clients/C.1000000000000000/vfs-update/ABCDEF" }, { "api_method": "GetVfsFileContentUpdateState", "method": "GET", - "response": { "message": "Operation with id ABCDEF not found" }, + "response": { + "message": "Operation with id ABCDEF not found" + }, "test_class": "ApiGetVfsFileContentUpdateStateHandlerRegressionTest_http_v2", "url": "/api/v2/clients/C.1000000000000000/vfs-update/ABCDEF" }, { "api_method": "GetVfsFileContentUpdateState", "method": "GET", - "response": { "message": "Flow id has incorrect format: `ABCDEF`" }, + "response": { + "message": "Flow id has incorrect format: `ABCDEF`" + }, "test_class": "ApiGetVfsFileContentUpdateStateHandlerRegressionTest_http_v2", "url": "/api/v2/clients/C.1000000000000000/vfs-update/ABCDEF" } @@ -2321,28 +3111,36 @@ { "api_method": "GetVfsRefreshOperationState", "method": "GET", - "response": { "state": "RUNNING" }, + "response": { + "state": "RUNNING" + }, "test_class": "ApiGetVfsRefreshOperationStateHandlerRegressionTest_http_v2", "url": "/api/v2/clients/C.1000000000000000/vfs-refresh-operations/ABCDEF" }, { "api_method": "GetVfsRefreshOperationState", "method": "GET", - "response": { "state": "FINISHED" }, + "response": { + "state": "FINISHED" + }, "test_class": "ApiGetVfsRefreshOperationStateHandlerRegressionTest_http_v2", "url": "/api/v2/clients/C.1000000000000000/vfs-refresh-operations/ABCDEF" }, { "api_method": "GetVfsRefreshOperationState", "method": "GET", - "response": { "message": "Operation with id ABCDEF not found" }, + "response": { + "message": "Operation with id ABCDEF not found" + }, "test_class": "ApiGetVfsRefreshOperationStateHandlerRegressionTest_http_v2", "url": "/api/v2/clients/C.1000000000000000/vfs-refresh-operations/ABCDEF" }, { "api_method": "GetVfsRefreshOperationState", "method": "GET", - "response": { "message": "Operation with id ABCDEF not found" }, + "response": { + "message": "Operation with id ABCDEF not found" + }, "test_class": "ApiGetVfsRefreshOperationStateHandlerRegressionTest_http_v2", "url": "/api/v2/clients/C.1000000000000000/vfs-refresh-operations/ABCDEF" } @@ -2355,27 +3153,27 @@ "items": [ { "action": "MODIFICATION", - "filePath": "fs/os/Users/\u4e2d\u56fd\u65b0\u95fb\u7f51\u65b0\u95fb\u4e2d/Shared/a.txt", + "filePath": "fs/os/Users/中国新闻网新闻中/Shared/a.txt", "timestamp": "4000000" }, { "action": "MODIFICATION", - "filePath": "fs/os/Users/\u4e2d\u56fd\u65b0\u95fb\u7f51\u65b0\u95fb\u4e2d/Shared/a.txt", + "filePath": "fs/os/Users/中国新闻网新闻中/Shared/a.txt", "timestamp": "3000000" }, { "action": "MODIFICATION", - "filePath": "fs/os/Users/\u4e2d\u56fd\u65b0\u95fb\u7f51\u65b0\u95fb\u4e2d/Shared/a.txt", + "filePath": "fs/os/Users/中国新闻网新闻中/Shared/a.txt", "timestamp": "2000000" }, { "action": "MODIFICATION", - "filePath": "fs/os/Users/\u4e2d\u56fd\u65b0\u95fb\u7f51\u65b0\u95fb\u4e2d/Shared/a.txt", + "filePath": "fs/os/Users/中国新闻网新闻中/Shared/a.txt", "timestamp": "1000000" }, { "action": "MODIFICATION", - "filePath": "fs/os/Users/\u4e2d\u56fd\u65b0\u95fb\u7f51\u65b0\u95fb\u4e2d/Shared/a.txt", + "filePath": "fs/os/Users/中国新闻网新闻中/Shared/a.txt", "timestamp": "0" } ] @@ -2389,12 +3187,17 @@ "api_method": "GrantClientApproval", "method": "POST", "response": { - "approvers": ["api_test_user", "requestor"], + "approvers": [ + "api_test_user", + "requestor" + ], "emailMessageId": "", "expirationTimeUs": "2419244000000", "id": "approval:111111", "isValid": true, - "notifiedUsers": ["api_test_user"], + "notifiedUsers": [ + "api_test_user" + ], "reason": "foo", "requestor": "requestor", "subject": { @@ -2403,10 +3206,12 @@ "buildTime": "1980-01-01", "clientName": "GRR Monitor", "clientVersion": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "clientId": "C.1000000000000000", - "fleetspeakEnabled": false, "hardwareInfo": { "biosVersion": "Bios-Version-0", "systemManufacturer": "System-Manufacturer-0" @@ -2414,7 +3219,10 @@ "interfaces": [ { "addresses": [ - { "addressType": "INET", "packedBytes": "wKgAAA==" }, + { + "addressType": "INET", + "packedBytes": "wKgAAA==" + }, { "addressType": "INET6", "packedBytes": "IAGrzQAAAAAAAAAAAAAAAA==" @@ -2422,13 +3230,26 @@ ], "ifname": "if0" }, - { "ifname": "if1", "macAddress": "qrvM3e4A" }, - { "ifname": "if2", "macAddress": "u8zd7v8A" } + { + "ifname": "if1", + "macAddress": "qrvM3e4A" + }, + { + "ifname": "if2", + "macAddress": "u8zd7v8A" + } ], "knowledgeBase": { "fqdn": "Host-0.example.com", "os": "Linux", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] }, "lastSeenAt": "42000000", "osInfo": { @@ -2439,7 +3260,14 @@ "version": "buster/sid" }, "urn": "aff4:/C.1000000000000000", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] } }, "test_class": "ApiGrantClientApprovalHandlerRegressionTest_http_v2", @@ -2451,11 +3279,16 @@ "api_method": "GrantCronJobApproval", "method": "POST", "response": { - "approvers": ["api_test_user", "requestor"], + "approvers": [ + "api_test_user", + "requestor" + ], "emailMessageId": "", "id": "approval:111111", "isValid": true, - "notifiedUsers": ["api_test_user"], + "notifiedUsers": [ + "api_test_user" + ], "reason": "foo", "requestor": "requestor", "subject": { @@ -2490,12 +3323,17 @@ "api_method": "GrantHuntApproval", "method": "POST", "response": { - "approvers": ["api_test_user", "requestor"], + "approvers": [ + "api_test_user", + "requestor" + ], "emailMessageId": "", "expirationTimeUs": "2419244000000", "id": "approval:111111", "isValid": true, - "notifiedUsers": ["api_test_user"], + "notifiedUsers": [ + "api_test_user" + ], "reason": "foo", "requestor": "requestor", "subject": { @@ -2505,7 +3343,10 @@ "clientRuleSet": { "rules": [ { - "regex": { "attributeRegex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attributeRegex": "GRR", + "field": "CLIENT_NAME" + }, "ruleType": "REGEX" } ] @@ -2521,7 +3362,10 @@ "failedClientsCount": "0", "flowArgs": { "@type": "type.googleapis.com/grr.GetFileArgs", - "pathspec": { "path": "/tmp/evil.txt", "pathtype": "OS" } + "pathspec": { + "path": "/tmp/evil.txt", + "pathtype": "OS" + } }, "flowName": "GetFile", "huntId": "H:123456", @@ -2533,7 +3377,10 @@ "clientRuleSet": { "rules": [ { - "regex": { "attributeRegex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attributeRegex": "GRR", + "field": "CLIENT_NAME" + }, "ruleType": "REGEX" } ] @@ -2551,6 +3398,7 @@ "resultsCount": "0", "state": "PAUSED", "stateComment": "", + "stateReason": "UNKNOWN", "totalCpuUsage": 0.0, "totalNetUsage": "0", "urn": "aff4:/hunts/H:123456" @@ -2584,7 +3432,9 @@ "message": "Host-0.example.com: ", "notificationType": "TYPE_CLIENT_INTERROGATED", "reference": { - "client": { "clientId": "C.1000000000000000" }, + "client": { + "clientId": "C.1000000000000000" + }, "type": "CLIENT" }, "timestamp": "42000000" @@ -2598,7 +3448,10 @@ { "api_method": "ListAndResetUserNotifications", "method": "POST", - "request_payload": { "count": "1", "offset": "1" }, + "request_payload": { + "count": "1", + "offset": "1" + }, "response": { "items": [ { @@ -2606,7 +3459,9 @@ "message": "Host-0.example.com: ", "notificationType": "TYPE_CLIENT_INTERROGATED", "reference": { - "client": { "clientId": "C.1000000000000000" }, + "client": { + "clientId": "C.1000000000000000" + }, "type": "CLIENT" }, "timestamp": "42000000" @@ -2620,7 +3475,9 @@ { "api_method": "ListAndResetUserNotifications", "method": "POST", - "request_payload": { "filter": "other" }, + "request_payload": { + "filter": "other" + }, "response": { "items": [ { @@ -2656,9 +3513,15 @@ "method": "GET", "response": { "suggestions": [ - { "username": "sanchezmorty" }, - { "username": "sanchezrick" }, - { "username": "sanchezsummer" } + { + "username": "sanchezmorty" + }, + { + "username": "sanchezrick" + }, + { + "username": "sanchezsummer" + } ] }, "test_class": "ApiListApproverSuggestionsHandlerRegressionTest_http_v2", @@ -2673,7 +3536,6 @@ "items": [ { "artifact": { - "conditions": ["os_major_version >= 6"], "doc": "Extract the installed drivers on Windows via WMI.", "name": "TestDrivers", "sources": [ @@ -2681,16 +3543,24 @@ "attributes": { "dat": [ { - "k": { "string": "query" }, - "v": { "string": "SELECT * from Win32_SystemDriver" } + "k": { + "string": "query" + }, + "v": { + "string": "SELECT * from Win32_SystemDriver" + } } ] }, "type": "WMI" } ], - "supportedOs": ["Windows"], - "urls": ["http://www.example.com"] + "supportedOs": [ + "Windows" + ], + "urls": [ + "http://www.example.com" + ] }, "errorMessage": "", "isCustom": false @@ -2706,50 +3576,14 @@ { "api_method": "ListClientActionRequests", "method": "GET", - "response": { - "items": [ - { - "clientAction": "ListProcesses", - "leasedUntil": "10042000000", - "sessionId": "aff4:/C.1000000000000000/flows/W:ABCDEF" - } - ] - }, + "response": {}, "test_class": "ApiListClientActionRequestsHandlerRegressionTest_http_v2", "url": "/api/v2/clients/C.1000000000000000/action-requests" }, { "api_method": "ListClientActionRequests", "method": "GET", - "response": { - "items": [ - { - "clientAction": "ListProcesses", - "leasedUntil": "10042000000", - "responses": [ - { - "args": "Ggx0ZXN0X3Byb2Nlc3M=", - "argsRdfName": "Process", - "requestId": "1", - "responseId": "1", - "sessionId": "aff4:/C.1000000000000000/flows/W:ABCDEF", - "timestamp": "42000000", - "type": "MESSAGE" - }, - { - "args": "CAA=", - "argsRdfName": "GrrStatus", - "requestId": "1", - "responseId": "2", - "sessionId": "aff4:/C.1000000000000000/flows/W:ABCDEF", - "timestamp": "42000000", - "type": "STATUS" - } - ], - "sessionId": "aff4:/C.1000000000000000/flows/W:ABCDEF" - } - ] - }, + "response": {}, "test_class": "ApiListClientActionRequestsHandlerRegressionTest_http_v2", "url": "/api/v2/clients/C.1000000000000000/action-requests?fetch_responses=1" } @@ -2761,12 +3595,17 @@ "response": { "items": [ { - "approvers": ["api_test_user", "approver"], + "approvers": [ + "api_test_user", + "approver" + ], "emailMessageId": "", "expirationTimeUs": "2419245000000", "id": "approval:222222", "isValid": true, - "notifiedUsers": ["approver"], + "notifiedUsers": [ + "approver" + ], "reason": "Running tests", "requestor": "api_test_user", "subject": { @@ -2775,10 +3614,12 @@ "buildTime": "1980-01-01", "clientName": "GRR Monitor", "clientVersion": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "clientId": "C.1000000000000001", - "fleetspeakEnabled": false, "hardwareInfo": { "biosVersion": "Bios-Version-1", "systemManufacturer": "System-Manufacturer-1" @@ -2786,7 +3627,10 @@ "interfaces": [ { "addresses": [ - { "addressType": "INET", "packedBytes": "wKgAAQ==" }, + { + "addressType": "INET", + "packedBytes": "wKgAAQ==" + }, { "addressType": "INET6", "packedBytes": "IAGrzQAAAAAAAAAAAAAAAQ==" @@ -2794,13 +3638,26 @@ ], "ifname": "if0" }, - { "ifname": "if1", "macAddress": "qrvM3e4B" }, - { "ifname": "if2", "macAddress": "u8zd7v8B" } + { + "ifname": "if1", + "macAddress": "qrvM3e4B" + }, + { + "ifname": "if2", + "macAddress": "u8zd7v8B" + } ], "knowledgeBase": { "fqdn": "Host-1.example.com", "os": "Linux", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] }, "lastSeenAt": "42000000", "osInfo": { @@ -2811,17 +3668,28 @@ "version": "buster/sid" }, "urn": "aff4:/C.1000000000000001", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] } }, { - "approvers": ["api_test_user"], + "approvers": [ + "api_test_user" + ], "emailMessageId": "", "expirationTimeUs": "2419244000000", "id": "approval:111111", "isValid": false, "isValidMessage": "Need at least 1 additional approver for access.", - "notifiedUsers": ["approver"], + "notifiedUsers": [ + "approver" + ], "reason": "Running tests", "requestor": "api_test_user", "subject": { @@ -2830,10 +3698,12 @@ "buildTime": "1980-01-01", "clientName": "GRR Monitor", "clientVersion": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "clientId": "C.1000000000000000", - "fleetspeakEnabled": false, "hardwareInfo": { "biosVersion": "Bios-Version-0", "systemManufacturer": "System-Manufacturer-0" @@ -2841,7 +3711,10 @@ "interfaces": [ { "addresses": [ - { "addressType": "INET", "packedBytes": "wKgAAA==" }, + { + "addressType": "INET", + "packedBytes": "wKgAAA==" + }, { "addressType": "INET6", "packedBytes": "IAGrzQAAAAAAAAAAAAAAAA==" @@ -2849,13 +3722,26 @@ ], "ifname": "if0" }, - { "ifname": "if1", "macAddress": "qrvM3e4A" }, - { "ifname": "if2", "macAddress": "u8zd7v8A" } + { + "ifname": "if1", + "macAddress": "qrvM3e4A" + }, + { + "ifname": "if2", + "macAddress": "u8zd7v8A" + } ], "knowledgeBase": { "fqdn": "Host-0.example.com", "os": "Linux", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] }, "lastSeenAt": "42000000", "osInfo": { @@ -2866,7 +3752,14 @@ "version": "buster/sid" }, "urn": "aff4:/C.1000000000000000", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] } } ] @@ -2880,13 +3773,17 @@ "response": { "items": [ { - "approvers": ["api_test_user"], + "approvers": [ + "api_test_user" + ], "emailMessageId": "", "expirationTimeUs": "2419244000000", "id": "approval:111111", "isValid": false, "isValidMessage": "Need at least 1 additional approver for access.", - "notifiedUsers": ["approver"], + "notifiedUsers": [ + "approver" + ], "reason": "Running tests", "requestor": "api_test_user", "subject": { @@ -2895,10 +3792,12 @@ "buildTime": "1980-01-01", "clientName": "GRR Monitor", "clientVersion": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "clientId": "C.1000000000000000", - "fleetspeakEnabled": false, "hardwareInfo": { "biosVersion": "Bios-Version-0", "systemManufacturer": "System-Manufacturer-0" @@ -2906,7 +3805,10 @@ "interfaces": [ { "addresses": [ - { "addressType": "INET", "packedBytes": "wKgAAA==" }, + { + "addressType": "INET", + "packedBytes": "wKgAAA==" + }, { "addressType": "INET6", "packedBytes": "IAGrzQAAAAAAAAAAAAAAAA==" @@ -2914,13 +3816,26 @@ ], "ifname": "if0" }, - { "ifname": "if1", "macAddress": "qrvM3e4A" }, - { "ifname": "if2", "macAddress": "u8zd7v8A" } + { + "ifname": "if1", + "macAddress": "qrvM3e4A" + }, + { + "ifname": "if2", + "macAddress": "u8zd7v8A" + } ], "knowledgeBase": { "fqdn": "Host-0.example.com", "os": "Linux", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] }, "lastSeenAt": "42000000", "osInfo": { @@ -2931,7 +3846,14 @@ "version": "buster/sid" }, "urn": "aff4:/C.1000000000000000", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] } } ] @@ -2952,7 +3874,10 @@ "buildTime": "1980-01-01", "clientName": "GRR Monitor", "clientVersion": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "crashMessage": "Client killed during transaction", "crashType": "Client Crash", @@ -2976,7 +3901,10 @@ "buildTime": "1980-01-01", "clientName": "GRR Monitor", "clientVersion": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "crashMessage": "Client killed during transaction", "crashType": "Client Crash", @@ -2992,7 +3920,9 @@ { "api_method": "ListClientCrashes", "method": "GET", - "response": { "totalCount": "1" }, + "response": { + "totalCount": "1" + }, "test_class": "ApiListClientCrashesHandlerRegressionTest_http_v2", "url": "/api/v2/clients/C.1000000000000000/crashes?count=1&offset=1" } @@ -3001,7 +3931,16 @@ { "api_method": "ListClientsLabels", "method": "GET", - "response": { "items": [{ "name": "bar" }, { "name": "foo" }] }, + "response": { + "items": [ + { + "name": "bar" + }, + { + "name": "foo" + } + ] + }, "test_class": "ApiListClientsLabelsHandlerRegressionTest_http_v2", "url": "/api/v2/clients/labels" } @@ -3171,7 +4110,10 @@ "items": [ { "argsType": "FileFinderArgs", - "behaviours": ["ADVANCED", "BASIC"], + "behaviours": [ + "ADVANCED", + "BASIC" + ], "category": "Filesystem", "defaultArgs": { "@type": "type.googleapis.com/grr.FileFinderArgs" @@ -3182,7 +4124,10 @@ }, { "argsType": "ListProcessesArgs", - "behaviours": ["ADVANCED", "BASIC"], + "behaviours": [ + "ADVANCED", + "BASIC" + ], "category": "Processes", "defaultArgs": { "@type": "type.googleapis.com/grr.ListProcessesArgs" @@ -3297,7 +4242,10 @@ { "payload": { "@type": "type.googleapis.com/grr.StatEntry", - "pathspec": { "path": "/tmp/evil.txt", "pathtype": "OS" }, + "pathspec": { + "path": "/tmp/evil.txt", + "pathtype": "OS" + }, "stAtime": "1336469177", "stCtime": "1336129892", "stDev": "64512", @@ -3386,7 +4334,9 @@ "progress": { "@type": "type.googleapis.com/grr.DefaultFlowProgress" }, - "resultMetadata": { "isMetadataSet": true }, + "resultMetadata": { + "isMetadataSet": true + }, "runnerArgs": { "clientId": "aff4:/C.1000000000000000", "flowName": "ListProcesses", @@ -3409,7 +4359,9 @@ "progress": { "@type": "type.googleapis.com/grr.DefaultFlowProgress" }, - "resultMetadata": { "isMetadataSet": true }, + "resultMetadata": { + "isMetadataSet": true + }, "runnerArgs": { "clientId": "aff4:/C.1000000000000000", "flowName": "Interrogate", @@ -3442,7 +4394,9 @@ "progress": { "@type": "type.googleapis.com/grr.DefaultFlowProgress" }, - "resultMetadata": { "isMetadataSet": true }, + "resultMetadata": { + "isMetadataSet": true + }, "runnerArgs": { "clientId": "aff4:/C.1000000000000000", "flowName": "ListProcesses", @@ -3475,7 +4429,9 @@ "progress": { "@type": "type.googleapis.com/grr.DefaultFlowProgress" }, - "resultMetadata": { "isMetadataSet": true }, + "resultMetadata": { + "isMetadataSet": true + }, "runnerArgs": { "clientId": "aff4:/C.1000000000000000", "flowName": "Interrogate", @@ -3550,13 +4506,17 @@ "response": { "items": [ { - "approvers": ["api_test_user"], + "approvers": [ + "api_test_user" + ], "emailMessageId": "", "expirationTimeUs": "2419243000000", "id": "approval:112233", "isValid": false, "isValidMessage": "Need at least 1 additional approver for access.", - "notifiedUsers": ["approver"], + "notifiedUsers": [ + "approver" + ], "reason": "Running tests", "requestor": "api_test_user", "subject": { @@ -3585,7 +4545,10 @@ "failedClientsCount": "0", "flowArgs": { "@type": "type.googleapis.com/grr.GetFileArgs", - "pathspec": { "path": "/tmp/evil.txt", "pathtype": "OS" } + "pathspec": { + "path": "/tmp/evil.txt", + "pathtype": "OS" + } }, "flowName": "GetFile", "huntId": "H:123456", @@ -3618,6 +4581,7 @@ "resultsCount": "0", "state": "PAUSED", "stateComment": "", + "stateReason": "UNKNOWN", "totalCpuUsage": 0.0, "totalNetUsage": "0", "urn": "aff4:/hunts/H:123456" @@ -3635,11 +4599,26 @@ "method": "GET", "response": { "items": [ - { "clientId": "C.1000000000000000", "flowId": "H:123456" }, - { "clientId": "C.1000000000000001", "flowId": "H:123456" }, - { "clientId": "C.1000000000000002", "flowId": "H:123456" }, - { "clientId": "C.1000000000000003", "flowId": "H:123456" }, - { "clientId": "C.1000000000000004", "flowId": "H:123456" } + { + "clientId": "C.1000000000000000", + "flowId": "H:123456" + }, + { + "clientId": "C.1000000000000001", + "flowId": "H:123456" + }, + { + "clientId": "C.1000000000000002", + "flowId": "H:123456" + }, + { + "clientId": "C.1000000000000003", + "flowId": "H:123456" + }, + { + "clientId": "C.1000000000000004", + "flowId": "H:123456" + } ], "totalCount": "5" }, @@ -3651,10 +4630,22 @@ "method": "GET", "response": { "items": [ - { "clientId": "C.1000000000000000", "flowId": "H:123456" }, - { "clientId": "C.1000000000000001", "flowId": "H:123456" }, - { "clientId": "C.1000000000000002", "flowId": "H:123456" }, - { "clientId": "C.1000000000000003", "flowId": "H:123456" } + { + "clientId": "C.1000000000000000", + "flowId": "H:123456" + }, + { + "clientId": "C.1000000000000001", + "flowId": "H:123456" + }, + { + "clientId": "C.1000000000000002", + "flowId": "H:123456" + }, + { + "clientId": "C.1000000000000003", + "flowId": "H:123456" + } ], "totalCount": "4" }, @@ -3665,7 +4656,12 @@ "api_method": "ListHuntClients", "method": "GET", "response": { - "items": [{ "clientId": "C.1000000000000004", "flowId": "H:123456" }], + "items": [ + { + "clientId": "C.1000000000000004", + "flowId": "H:123456" + } + ], "totalCount": "1" }, "test_class": "ApiListHuntClientsHandlerRegressionTest_http_v2", @@ -3684,7 +4680,10 @@ "buildTime": "1980-01-01", "clientName": "GRR Monitor", "clientVersion": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "crashMessage": "Client killed during transaction", "crashType": "Client Crash", @@ -3708,7 +4707,10 @@ "buildTime": "1980-01-01", "clientName": "GRR Monitor", "clientVersion": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "crashMessage": "Client killed during transaction", "crashType": "Client Crash", @@ -3724,7 +4726,9 @@ { "api_method": "ListHuntCrashes", "method": "GET", - "response": { "totalCount": "1" }, + "response": { + "totalCount": "1" + }, "test_class": "ApiListHuntCrashesHandlerRegressionTest_http_v2", "url": "/api/v2/hunts/H:123456/crashes?count=1&offset=1" } @@ -4297,7 +5301,9 @@ "message": "Host-0.example.com: ", "notificationType": "TYPE_CLIENT_INTERROGATED", "reference": { - "client": { "clientId": "C.1000000000000000" }, + "client": { + "clientId": "C.1000000000000000" + }, "type": "CLIENT" }, "timestamp": "42000000" @@ -4385,7 +5391,9 @@ { "api_method": "ModifyCronJob", "method": "PATCH", - "request_payload": { "enabled": true }, + "request_payload": { + "enabled": true + }, "response": { "allowOverruns": false, "args": { @@ -4415,7 +5423,9 @@ { "api_method": "ModifyCronJob", "method": "PATCH", - "request_payload": { "enabled": false }, + "request_payload": { + "enabled": false + }, "response": { "allowOverruns": false, "args": { @@ -4447,7 +5457,9 @@ { "api_method": "ModifyHunt", "method": "PATCH", - "request_payload": { "clientLimit": "142" }, + "request_payload": { + "clientLimit": "142" + }, "response": { "allClientsCount": "0", "clientLimit": "142", @@ -4455,7 +5467,10 @@ "clientRuleSet": { "rules": [ { - "regex": { "attributeRegex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attributeRegex": "GRR", + "field": "CLIENT_NAME" + }, "ruleType": "REGEX" } ] @@ -4471,7 +5486,10 @@ "failedClientsCount": "0", "flowArgs": { "@type": "type.googleapis.com/grr.GetFileArgs", - "pathspec": { "path": "/tmp/evil.txt", "pathtype": "OS" } + "pathspec": { + "path": "/tmp/evil.txt", + "pathtype": "OS" + } }, "flowName": "GetFile", "huntId": "H:123456", @@ -4484,7 +5502,10 @@ "clientRuleSet": { "rules": [ { - "regex": { "attributeRegex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attributeRegex": "GRR", + "field": "CLIENT_NAME" + }, "ruleType": "REGEX" } ] @@ -4502,6 +5523,7 @@ "resultsCount": "0", "state": "PAUSED", "stateComment": "", + "stateReason": "UNKNOWN", "totalCpuUsage": 0.0, "totalNetUsage": "0", "urn": "aff4:/hunts/H:123456" @@ -4512,7 +5534,9 @@ { "api_method": "ModifyHunt", "method": "PATCH", - "request_payload": { "state": "STOPPED" }, + "request_payload": { + "state": "STOPPED" + }, "response": { "allClientsCount": "0", "clientLimit": "142", @@ -4520,7 +5544,10 @@ "clientRuleSet": { "rules": [ { - "regex": { "attributeRegex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attributeRegex": "GRR", + "field": "CLIENT_NAME" + }, "ruleType": "REGEX" } ] @@ -4536,7 +5563,10 @@ "failedClientsCount": "0", "flowArgs": { "@type": "type.googleapis.com/grr.GetFileArgs", - "pathspec": { "path": "/tmp/evil.txt", "pathtype": "OS" } + "pathspec": { + "path": "/tmp/evil.txt", + "pathtype": "OS" + } }, "flowName": "GetFile", "huntId": "H:123456", @@ -4549,7 +5579,10 @@ "clientRuleSet": { "rules": [ { - "regex": { "attributeRegex": "GRR", "field": "CLIENT_NAME" }, + "regex": { + "attributeRegex": "GRR", + "field": "CLIENT_NAME" + }, "ruleType": "REGEX" } ] @@ -4567,6 +5600,7 @@ "resultsCount": "0", "state": "STOPPED", "stateComment": "Cancelled by user", + "stateReason": "TRIGGERED_BY_USER", "totalCpuUsage": 0.0, "totalNetUsage": "0", "urn": "aff4:/hunts/H:123456" @@ -4587,10 +5621,12 @@ "buildTime": "1980-01-01", "clientName": "GRR Monitor", "clientVersion": 1234, - "labels": ["label1", "label2"] + "labels": [ + "label1", + "label2" + ] }, "clientId": "C.1000000000000000", - "fleetspeakEnabled": false, "hardwareInfo": { "biosVersion": "Bios-Version-0", "systemManufacturer": "System-Manufacturer-0" @@ -4598,7 +5634,10 @@ "interfaces": [ { "addresses": [ - { "addressType": "INET", "packedBytes": "wKgAAA==" }, + { + "addressType": "INET", + "packedBytes": "wKgAAA==" + }, { "addressType": "INET6", "packedBytes": "IAGrzQAAAAAAAAAAAAAAAA==" @@ -4606,13 +5645,26 @@ ], "ifname": "if0" }, - { "ifname": "if1", "macAddress": "qrvM3e4A" }, - { "ifname": "if2", "macAddress": "u8zd7v8A" } + { + "ifname": "if1", + "macAddress": "qrvM3e4A" + }, + { + "ifname": "if2", + "macAddress": "u8zd7v8A" + } ], "knowledgeBase": { "fqdn": "Host-0.example.com", "os": "Linux", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] }, "lastSeenAt": "42000000", "osInfo": { @@ -4623,7 +5675,14 @@ "version": "buster/sid" }, "urn": "aff4:/C.1000000000000000", - "users": [{ "username": "user1" }, { "username": "user2" }] + "users": [ + { + "username": "user1" + }, + { + "username": "user2" + } + ] } ] }, @@ -4635,10 +5694,14 @@ { "api_method": "UpdateVfsFileContent", "method": "POST", - "request_payload": { "filePath": "fs/os/c/bin/bash" }, - "response": { "operationId": "ABCDEF" }, + "request_payload": { + "filePath": "fs/os/c/bin/bash" + }, + "response": { + "operationId": "ABCDEF" + }, "test_class": "ApiUpdateVfsFileContentHandlerRegressionTest_http_v2", "url": "/api/v2/clients/C.1000000000000000/vfs-update" } ] -} +} \ No newline at end of file diff --git a/grr/server/grr_response_server/gui/static/angular-components/forms/output-plugin-descriptor-form-directive.js b/grr/server/grr_response_server/gui/static/angular-components/forms/output-plugin-descriptor-form-directive.js index 4d87edd32f..e6721a9366 100644 --- a/grr/server/grr_response_server/gui/static/angular-components/forms/output-plugin-descriptor-form-directive.js +++ b/grr/server/grr_response_server/gui/static/angular-components/forms/output-plugin-descriptor-form-directive.js @@ -61,9 +61,7 @@ const OutputPluginDescriptorFormController = class { if (angular.isDefined(newValue)) { const argsType = this.outputPluginsDescriptors[newValue]['args_type']; - // Prefer reading `args` and fallback to `plugin_args` - const pluginArgs = this.scope_['value']['value']['args'] || - this.scope_['value']['value']['plugin_args']; + const pluginArgs = this.scope_['value']['value']['args']; // We want to replace the plugin args only if they're undefined or // their type differs from the selected ones. This check helps diff --git a/grr/server/grr_response_server/gui/static/angular-components/hunt/new-hunt-wizard/form-directive.js b/grr/server/grr_response_server/gui/static/angular-components/hunt/new-hunt-wizard/form-directive.js index cc67c1fbd3..535d61de95 100644 --- a/grr/server/grr_response_server/gui/static/angular-components/hunt/new-hunt-wizard/form-directive.js +++ b/grr/server/grr_response_server/gui/static/angular-components/hunt/new-hunt-wizard/form-directive.js @@ -6,9 +6,9 @@ const {ReflectionService} = goog.require('grrUi.core.reflectionService'); /** @const {string} */ -exports.DEFAULT_PLUGIN_URL = '/config/' + - 'AdminUI.new_hunt_wizard.default_output_plugin'; -var DEFAULT_PLUGIN_URL = exports.DEFAULT_PLUGIN_URL; +exports.DEFAULT_PLUGINS_URL = '/config/' + + 'AdminUI.new_hunt_wizard.default_output_plugins'; +const DEFAULT_PLUGINS_URL = exports.DEFAULT_PLUGINS_URL; @@ -37,15 +37,15 @@ const FormController = class { this.descriptors_ = {}; /** @private {?string} */ - this.defaultOutputPluginName; + this.defaultOutputPluginNames; /** @type {boolean} */ this.configureFlowPageHasErrors; - this.grrApiService_.get(DEFAULT_PLUGIN_URL) + this.grrApiService_.get(DEFAULT_PLUGINS_URL) .then(function(response) { if (angular.isDefined(response['data']['value'])) { - this.defaultOutputPluginName = response['data']['value']['value']; + this.defaultOutputPluginNames = response['data']['value']['value']; } return this.grrReflectionService_.getRDFValueDescriptor( @@ -83,15 +83,17 @@ const FormController = class { } if (angular.isUndefined(hra['value']['output_plugins'])) { - if (this.defaultOutputPluginName) { - var defaultPluginDescriptor = angular.copy( - this.descriptors_['OutputPluginDescriptor']['default']); - defaultPluginDescriptor['value']['plugin_name'] = - angular.copy(this.descriptors_['RDFString']['default']); - defaultPluginDescriptor['value']['plugin_name']['value'] = - this.defaultOutputPluginName; - - hra['value']['output_plugins'] = [defaultPluginDescriptor]; + if (this.defaultOutputPluginNames) { + hra['value']['output_plugins'] = []; + this.defaultOutputPluginNames.split(',').forEach((n) => { + var defaultPluginDescriptor = angular.copy( + this.descriptors_['OutputPluginDescriptor']['default']); + defaultPluginDescriptor['value']['plugin_name'] = + angular.copy(this.descriptors_['RDFString']['default']); + defaultPluginDescriptor['value']['plugin_name']['value'] = n; + + hra['value']['output_plugins'].push(defaultPluginDescriptor); + }); } else if (angular.isUndefined(newValue['value']['output_plugins'])) { hra['value']['output_plugins'] = []; } diff --git a/grr/server/grr_response_server/gui/ui/components/app/app.ng.html b/grr/server/grr_response_server/gui/ui/components/app/app.ng.html index a63f573bea..db7f81e4ca 100644 --- a/grr/server/grr_response_server/gui/ui/components/app/app.ng.html +++ b/grr/server/grr_response_server/gui/ui/components/app/app.ng.html @@ -10,7 +10,7 @@ -