diff --git a/.env b/.env index 59f7df83..c62498b0 100644 --- a/.env +++ b/.env @@ -11,6 +11,6 @@ scheme=https # Your version of Splunk (default: 6.2) version=9.0 # Bearer token for authentication -#bearerToken="" +#splunkToken="" # Session key for authentication -#sessionKey="" +#token="" diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml new file mode 100644 index 00000000..9df99942 --- /dev/null +++ b/.github/dependabot.yaml @@ -0,0 +1,7 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + target-branch: "master" + schedule: + interval: "weekly" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4d11da59..9309a311 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -29,12 +29,11 @@ jobs: run: | rm -rf ./docs/_build tox -e docs - cd ./docs/_build/html && zip -r ../docs_html.zip . -x ".*" -x "__MACOSX" - name : Docs Upload uses: actions/upload-artifact@v3 with: - name: apidocs - path: docs/_build/docs_html.zip + name: python_sdk_docs + path: docs/_build/html # Test upload # - name: Publish package to TestPyPI # uses: pypa/gh-action-pypi-publish@master diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 06278e29..560e8bc0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,6 +13,7 @@ jobs: - ubuntu-latest python: [ 2.7, 3.7 ] splunk-version: + - "8.1" - "8.2" - "latest" fail-fast: false diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ceda287..f7d56752 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Splunk Enterprise SDK for Python Changelog +## Version 1.7.3 + +### Bug fixes +* [#493](https://github.com/splunk/splunk-sdk-python/pull/493) Fixed file permission for event_writer.py file [[issue#487](https://github.com/splunk/splunk-sdk-python/issues/487)] +* [#500](https://github.com/splunk/splunk-sdk-python/pull/500) Replaced index_field with accelerated_field for kvstore [[issue#497](https://github.com/splunk/splunk-sdk-python/issues/497)] +* [#502](https://github.com/splunk/splunk-sdk-python/pull/502) Updated check for IPv6 addresses + +### Minor changes +* [#490](https://github.com/splunk/splunk-sdk-python/pull/490) Added ACL properties update feature +* [#495](https://github.com/splunk/splunk-sdk-python/pull/495) Added Splunk 8.1 in GitHub Actions Matrix +* [#485](https://github.com/splunk/splunk-sdk-python/pull/485) Added test case for cookie persistence +* [#503](https://github.com/splunk/splunk-sdk-python/pull/503) README updates on accessing "service" instance in CSC and ModularInput apps +* [#504](https://github.com/splunk/splunk-sdk-python/pull/504) Updated authentication token names in docs to reduce confusion +* [#494](https://github.com/splunk/splunk-sdk-python/pull/494) Reuse splunklib.__version__ in handler.request + ## Version 1.7.2 ### Minor changes diff --git a/README.md b/README.md index 29b75704..e28232d9 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # The Splunk Enterprise Software Development Kit for Python -#### Version 1.7.2 +#### Version 1.7.3 The Splunk Enterprise Software Development Kit (SDK) for Python contains library code designed to enable developers to build applications using the Splunk platform. @@ -30,7 +30,7 @@ Here's what you need to get going with the Splunk Enterprise SDK for Python. * Splunk Enterprise 9.0 or 8.2 - The Splunk Enterprise SDK for Python has been tested with Splunk Enterprise 9.0 and 8.2 + The Splunk Enterprise SDK for Python has been tested with Splunk Enterprise 9.0, 8.2 and 8.1 If you haven't already installed Splunk Enterprise, download it [here](http://www.splunk.com/download). For more information, see the Splunk Enterprise [_Installation Manual_](https://docs.splunk.com/Documentation/Splunk/latest/Installation). @@ -111,9 +111,9 @@ here is an example of .env file: # Your version of Splunk Enterprise version=9.0 # Bearer token for authentication - #bearerToken= + #splunkToken= # Session key for authentication - #sessionKey= + #token= #### SDK examples @@ -209,7 +209,39 @@ class GeneratorTest(GeneratingCommand): checkpoint_dir = inputs.metadata["checkpoint_dir"] ``` -#### Optional:Set up logging for splunklib +### Access service object in Custom Search Command & Modular Input apps + +#### Custom Search Commands +* The service object is created from the Splunkd URI and session key passed to the command invocation the search results info file. +* Service object can be accessed using `self.service` in `generate`/`transform`/`stream`/`reduce` methods depending on the Custom Search Command. +* For Generating Custom Search Command + ```python + def generate(self): + # other code + + # access service object that can be used to connect Splunk Service + service = self.service + # to get Splunk Service Info + info = service.info + ``` + + + +#### Modular Inputs app: +* The service object is created from the Splunkd URI and session key passed to the command invocation on the modular input stream respectively. +* It is available as soon as the `Script.stream_events` method is called. +```python + def stream_events(self, inputs, ew): + # other code + + # access service object that can be used to connect Splunk Service + service = self.service + # to get Splunk Service Info + info = service.info +``` + + +### Optional:Set up logging for splunklib + The default level is WARNING, which means that only events of this level and above will be visible + To change a logging level we can call setup_logging() method and pass the logging level as an argument. + Optional: we can also pass log format and date format string as a method argument to modify default format diff --git a/scripts/templates/env.template b/scripts/templates/env.template index a45851b6..ac9ebe5c 100644 --- a/scripts/templates/env.template +++ b/scripts/templates/env.template @@ -11,6 +11,6 @@ scheme=$scheme # Your version of Splunk (default: 6.2) version=$version # Bearer token for authentication -#bearerToken= +#splunkToken= # Session key for authentication -#sessionKey= \ No newline at end of file +#token= \ No newline at end of file diff --git a/splunklib/__init__.py b/splunklib/__init__.py index 774cb757..31787bdc 100644 --- a/splunklib/__init__.py +++ b/splunklib/__init__.py @@ -31,5 +31,5 @@ def setup_logging(level, log_format=DEFAULT_LOG_FORMAT, date_format=DEFAULT_DATE format=log_format, datefmt=date_format) -__version_info__ = (1, 7, 2) +__version_info__ = (1, 7, 3) __version__ = ".".join(map(str, __version_info__)) diff --git a/splunklib/binding.py b/splunklib/binding.py index 17b5783c..85cb8d12 100644 --- a/splunklib/binding.py +++ b/splunklib/binding.py @@ -39,6 +39,7 @@ from io import BytesIO from xml.etree.ElementTree import XML +from splunklib import __version__ from splunklib import six from splunklib.six.moves import urllib @@ -346,7 +347,8 @@ def _authority(scheme=DEFAULT_SCHEME, host=DEFAULT_HOST, port=DEFAULT_PORT): "http://splunk.utopia.net:471" """ - if ':' in host: + # check if host is an IPv6 address and not enclosed in [ ] + if ':' in host and not (host.startswith('[') and host.endswith(']')): # IPv6 addresses must be enclosed in [ ] in order to be well # formed. host = '[' + host + ']' @@ -1434,7 +1436,7 @@ def request(url, message, **kwargs): head = { "Content-Length": str(len(body)), "Host": host, - "User-Agent": "splunk-sdk-python/1.7.2", + "User-Agent": "splunk-sdk-python/%s" % __version__, "Accept": "*/*", "Connection": "Close", } # defaults diff --git a/splunklib/client.py b/splunklib/client.py index cde39e95..33156bb5 100644 --- a/splunklib/client.py +++ b/splunklib/client.py @@ -1216,6 +1216,36 @@ def reload(self): self.post("_reload") return self + def acl_update(self, **kwargs): + """To update Access Control List (ACL) properties for an endpoint. + + :param kwargs: Additional entity-specific arguments (required). + + - "owner" (``string``): The Splunk username, such as "admin". A value of "nobody" means no specific user (required). + + - "sharing" (``string``): A mode that indicates how the resource is shared. The sharing mode can be "user", "app", "global", or "system" (required). + + :type kwargs: ``dict`` + + **Example**:: + + import splunklib.client as client + service = client.connect(...) + saved_search = service.saved_searches["name"] + saved_search.acl_update(sharing="app", owner="nobody", app="search", **{"perms.read": "admin, nobody"}) + """ + if "body" not in kwargs: + kwargs = {"body": kwargs} + + if "sharing" not in kwargs["body"]: + raise ValueError("Required argument 'sharing' is missing.") + if "owner" not in kwargs["body"]: + raise ValueError("Required argument 'owner' is missing.") + + self.post("acl", **kwargs) + self.refresh() + return self + @property def state(self): """Returns the entity's state record. @@ -3679,13 +3709,20 @@ class KVStoreCollections(Collection): def __init__(self, service): Collection.__init__(self, service, 'storage/collections/config', item=KVStoreCollection) - def create(self, name, indexes = {}, fields = {}, **kwargs): + def __getitem__(self, item): + res = Collection.__getitem__(self, item) + for k, v in res.content.items(): + if "accelerated_fields" in k: + res.content[k] = json.loads(v) + return res + + def create(self, name, accelerated_fields={}, fields={}, **kwargs): """Creates a KV Store Collection. :param name: name of collection to create :type name: ``string`` - :param indexes: dictionary of index definitions - :type indexes: ``dict`` + :param accelerated_fields: dictionary of accelerated_fields definitions + :type accelerated_fields: ``dict`` :param fields: dictionary of field definitions :type fields: ``dict`` :param kwargs: a dictionary of additional parameters specifying indexes and field definitions @@ -3693,10 +3730,10 @@ def create(self, name, indexes = {}, fields = {}, **kwargs): :return: Result of POST request """ - for k, v in six.iteritems(indexes): + for k, v in six.iteritems(accelerated_fields): if isinstance(v, dict): v = json.dumps(v) - kwargs['index.' + k] = v + kwargs['accelerated_fields.' + k] = v for k, v in six.iteritems(fields): kwargs['field.' + k] = v return self.post(name=name, **kwargs) @@ -3710,18 +3747,20 @@ def data(self): """ return KVStoreCollectionData(self) - def update_index(self, name, value): - """Changes the definition of a KV Store index. + def update_accelerated_field(self, name, value): + """Changes the definition of a KV Store accelerated_field. - :param name: name of index to change + :param name: name of accelerated_fields to change :type name: ``string`` - :param value: new index definition - :type value: ``dict`` or ``string`` + :param value: new accelerated_fields definition + :type value: ``dict`` :return: Result of POST request """ kwargs = {} - kwargs['index.' + name] = value if isinstance(value, six.string_types) else json.dumps(value) + if isinstance(value, dict): + value = json.dumps(value) + kwargs['accelerated_fields.' + name] = value return self.post(**kwargs) def update_field(self, name, value): diff --git a/splunklib/modularinput/event_writer.py b/splunklib/modularinput/event_writer.py old mode 100755 new mode 100644 diff --git a/tests/test_binding.py b/tests/test_binding.py index c101b19c..2af294cf 100755 --- a/tests/test_binding.py +++ b/tests/test_binding.py @@ -190,6 +190,12 @@ def test_ipv6_host(self): host="2001:0db8:85a3:0000:0000:8a2e:0370:7334"), "https://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:8089") + def test_ipv6_host_enclosed(self): + self.assertEqual( + binding._authority( + host="[2001:0db8:85a3:0000:0000:8a2e:0370:7334]"), + "https://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:8089") + def test_all_fields(self): self.assertEqual( binding._authority( @@ -491,6 +497,56 @@ def test_handlers(self): body = context.get(path).body.read() self.assertTrue(isatom(body)) +def urllib2_insert_cookie_handler(url, message, **kwargs): + method = message['method'].lower() + data = message.get('body', b"") if method == 'post' else None + headers = dict(message.get('headers', [])) + req = Request(url, data, headers) + try: + # If running Python 2.7.9+, disable SSL certificate validation + if sys.version_info >= (2, 7, 9): + response = urlopen(req, context=ssl._create_unverified_context()) + else: + response = urlopen(req) + except HTTPError as response: + pass # Propagate HTTP errors via the returned response message + + # Mimic the insertion of 3rd party cookies into the response. + # An example is "sticky session"/"insert cookie" persistence + # of a load balancer for a SHC. + header_list = [(k, v) for k, v in response.info().items()] + header_list.append(("Set-Cookie", "BIGipServer_splunk-shc-8089=1234567890.12345.0000; path=/; Httponly; Secure")) + header_list.append(("Set-Cookie", "home_made=yummy")) + + return { + 'status': response.code, + 'reason': response.msg, + 'headers': header_list, + 'body': BytesIO(response.read()) + } + +class TestCookiePersistence(testlib.SDKTestCase): + # Verify persistence of 3rd party inserted cookies. + def test_3rdPartyInsertedCookiePersistence(self): + paths = ["/services", "authentication/users", + "search/jobs"] + logging.debug("Connecting with urllib2_insert_cookie_handler %s", urllib2_insert_cookie_handler) + context = binding.connect( + handler=urllib2_insert_cookie_handler, + **self.opts.kwargs) + + persisted_cookies = context.get_cookies() + + splunk_token_found = False + for k, v in persisted_cookies.items(): + if k[:8] == "splunkd_": + splunk_token_found = True + break + + self.assertEqual(splunk_token_found, True) + self.assertEqual(persisted_cookies['BIGipServer_splunk-shc-8089'], "1234567890.12345.0000") + self.assertEqual(persisted_cookies['home_made'], "yummy") + @pytest.mark.smoke class TestLogout(BindingTestCase): def test_logout(self): diff --git a/tests/test_kvstore_conf.py b/tests/test_kvstore_conf.py index a2453728..98020342 100755 --- a/tests/test_kvstore_conf.py +++ b/tests/test_kvstore_conf.py @@ -15,6 +15,8 @@ # under the License. from __future__ import absolute_import + +import json from tests import testlib try: import unittest @@ -42,13 +44,27 @@ def test_create_delete_collection(self): self.confs['test'].delete() self.assertTrue(not 'test' in self.confs) + def test_create_fields(self): + self.confs.create('test', accelerated_fields={'ind1':{'a':1}}, fields={'a':'number1'}) + self.assertEqual(self.confs['test']['field.a'], 'number1') + self.assertEqual(self.confs['test']['accelerated_fields.ind1'], {"a": 1}) + self.confs['test'].delete() + def test_update_collection(self): self.confs.create('test') - self.confs['test'].post(**{'accelerated_fields.ind1': '{"a": 1}', 'field.a': 'number'}) + val = {"a": 1} + self.confs['test'].post(**{'accelerated_fields.ind1': json.dumps(val), 'field.a': 'number'}) self.assertEqual(self.confs['test']['field.a'], 'number') - self.assertEqual(self.confs['test']['accelerated_fields.ind1'], '{"a": 1}') + self.assertEqual(self.confs['test']['accelerated_fields.ind1'], {"a": 1}) self.confs['test'].delete() + def test_update_accelerated_fields(self): + self.confs.create('test', accelerated_fields={'ind1':{'a':1}}) + self.assertEqual(self.confs['test']['accelerated_fields.ind1'], {'a': 1}) + # update accelerated_field value + self.confs['test'].update_accelerated_field('ind1', {'a': -1}) + self.assertEqual(self.confs['test']['accelerated_fields.ind1'], {'a': -1}) + self.confs['test'].delete() def test_update_fields(self): self.confs.create('test') @@ -77,17 +93,6 @@ def test_overlapping_collections(self): self.confs['test'].delete() self.confs['test'].delete() - """ - def test_create_accelerated_fields_fields(self): - self.confs.create('test', indexes={'foo': '{"foo": 1}', 'bar': {'bar': -1}}, **{'field.foo': 'string'}) - self.assertEqual(self.confs['test']['accelerated_fields.foo'], '{"foo": 1}') - self.assertEqual(self.confs['test']['field.foo'], 'string') - self.assertRaises(client.HTTPError, lambda: self.confs['test'].post(**{'accelerated_fields.foo': 'THIS IS INVALID'})) - self.assertEqual(self.confs['test']['accelerated_fields.foo'], '{"foo": 1}') - self.confs['test'].update_accelerated_fields('foo', '') - self.assertEqual(self.confs['test']['accelerated_fields.foo'], None) - """ - def tearDown(self): if ('test' in self.confs): self.confs['test'].delete() diff --git a/tests/test_saved_search.py b/tests/test_saved_search.py index c15921c0..d1f8f57c 100755 --- a/tests/test_saved_search.py +++ b/tests/test_saved_search.py @@ -223,6 +223,30 @@ def test_suppress(self): self.saved_search.unsuppress() self.assertEqual(self.saved_search['suppressed'], 0) + def test_acl(self): + self.assertEqual(self.saved_search.access["perms"], None) + self.saved_search.acl_update(sharing="app", owner="admin", app="search", **{"perms.read": "admin, nobody"}) + self.assertEqual(self.saved_search.access["owner"], "admin") + self.assertEqual(self.saved_search.access["app"], "search") + self.assertEqual(self.saved_search.access["sharing"], "app") + self.assertEqual(self.saved_search.access["perms"]["read"], ['admin', 'nobody']) + + def test_acl_fails_without_sharing(self): + self.assertRaisesRegex( + ValueError, + "Required argument 'sharing' is missing.", + self.saved_search.acl_update, + owner="admin", app="search", **{"perms.read": "admin, nobody"} + ) + + def test_acl_fails_without_owner(self): + self.assertRaisesRegex( + ValueError, + "Required argument 'owner' is missing.", + self.saved_search.acl_update, + sharing="app", app="search", **{"perms.read": "admin, nobody"} + ) + if __name__ == "__main__": try: import unittest2 as unittest diff --git a/tests/test_utils.py b/tests/test_utils.py index 5b6b712c..5eedbaba 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,20 +1,24 @@ from __future__ import absolute_import from tests import testlib +import unittest +import os +import sys + try: from utils import * except ImportError: raise Exception("Add the SDK repository to your PYTHONPATH to run the test cases " "(e.g., export PYTHONPATH=~/splunk-sdk-python.") - TEST_DICT = { - 'username':'admin', - 'password':'changeme', - 'port' : 8089, - 'host' : 'localhost', - 'scheme': 'https' - } + 'username': 'admin', + 'password': 'changeme', + 'port': 8089, + 'host': 'localhost', + 'scheme': 'https' +} + class TestUtils(testlib.SDKTestCase): def setUp(self): @@ -23,16 +27,16 @@ def setUp(self): # Test dslice when a dict is passed to change key names def test_dslice_dict_args(self): args = { - 'username':'user-name', - 'password':'new_password', - 'port': 'admin_port', - 'foo':'bar' - } + 'username': 'user-name', + 'password': 'new_password', + 'port': 'admin_port', + 'foo': 'bar' + } expected = { - 'user-name':'admin', - 'new_password':'changeme', - 'admin_port':8089 - } + 'user-name': 'admin', + 'new_password': 'changeme', + 'admin_port': 8089 + } self.assertTrue(expected == dslice(TEST_DICT, args)) # Test dslice when a list is passed @@ -43,40 +47,66 @@ def test_dslice_list_args(self): 'port', 'host', 'foo' - ] + ] expected = { - 'username':'admin', - 'password':'changeme', - 'port':8089, - 'host':'localhost' - } + 'username': 'admin', + 'password': 'changeme', + 'port': 8089, + 'host': 'localhost' + } self.assertTrue(expected == dslice(TEST_DICT, test_list)) # Test dslice when a single string is passed def test_dslice_arg(self): test_arg = 'username' expected = { - 'username':'admin' - } + 'username': 'admin' + } self.assertTrue(expected == dslice(TEST_DICT, test_arg)) # Test dslice using all three types of arguments def test_dslice_all_args(self): test_args = [ - {'username':'new_username'}, + {'username': 'new_username'}, ['password', - 'host'], + 'host'], 'port' ] expected = { - 'new_username':'admin', - 'password':'changeme', - 'host':'localhost', - 'port':8089 + 'new_username': 'admin', + 'password': 'changeme', + 'host': 'localhost', + 'port': 8089 } self.assertTrue(expected == dslice(TEST_DICT, *test_args)) +class FilePermissionTest(unittest.TestCase): + + def setUp(self): + super(FilePermissionTest, self).setUp() + + # Check for any change in the default file permission(i.e 644) for all files within splunklib + def test_filePermissions(self): + + def checkFilePermissions(dir_path): + for file in os.listdir(dir_path): + if file.__contains__('pycache'): + continue + path = os.path.join(dir_path, file) + if os.path.isfile(path): + permission = oct(os.stat(path).st_mode) + if sys.version_info >= (3, 0): + self.assertEqual(permission, '0o100644') + else : + self.assertEqual(permission, '0100644') + else: + checkFilePermissions(path) + + dir_path = os.path.join('..', 'splunklib') + checkFilePermissions(dir_path) + + if __name__ == "__main__": try: import unittest2 as unittest