Merge branch 'opensearch-project:main' into skip/test

.github/workflows/ci.yml

@@ -18,6 +18,21 @@ jobs:
           python3.7 -m pip install nox
       - name: Lint the code
         run: nox -s lint
+
+  generate:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v3
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.7
+      - name: Install dependencies
+        run: |
+          python3.7 -m pip install nox
+      - name: Run the api generator
+        run: nox -s generate
 
   test-build-distribution:
     runs-on: ubuntu-latest

.pylintrc

@@ -0,0 +1,15 @@
+[MESSAGES CONTROL]
+disable=all
+enable=line-too-long,
+    invalid-name,
+    pointless-statement,
+    unspecified-encoding,
+    missing-function-docstring,
+    missing-param-doc,
+    differing-param-doc,
+    unnecessary-dunder-call,
+    assignment-from-no-return,
+    unused-variable
+max-line-length=240
+good-names-rgxs=^[_a-z][_a-z0-9]?$
+

CHANGELOG.md

@@ -3,12 +3,18 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ## [Unreleased]
 ### Added
+- Added pylint `assignment-from-no-return` and `unused-variable` (([#658](https://github.com/opensearch-project/opensearch-py/pull/658))
+- Added pylint `unnecessary-dunder-calls` (([#655](https://github.com/opensearch-project/opensearch-py/pull/655))
+- Changed to use .pylintrc files in root and any directory with override requirements (([#654](https://github.com/opensearch-project/opensearch-py/pull/654))
+- Added pylint `unspecified-encoding` and `missing-function-docstring` and ignored opensearchpy for lints (([#643](https://github.com/opensearch-project/opensearch-py/pull/643)))
 - Added pylint `line-too-long` and `invalid-name` ([#590](https://github.com/opensearch-project/opensearch-py/pull/590))
 - Added pylint `pointless-statement` ([#611](https://github.com/opensearch-project/opensearch-py/pull/611))
 - Added a log collection guide ([#579](https://github.com/opensearch-project/opensearch-py/pull/579))
 - Added GHA release ([#614](https://github.com/opensearch-project/opensearch-py/pull/614))
+- Incorporated API generation into CI workflow and fixed 'generate' nox session ([#660](https://github.com/opensearch-project/opensearch-py/pull/660))
 ### Changed
 - Updated the `get_policy` API in the index_management plugin to allow the policy_id argument as optional ([#633](https://github.com/opensearch-project/opensearch-py/pull/633))
+- Updated the `point_in_time.md` guide with examples demonstrating the usage of the new APIs as alternatives to the deprecated ones. ([#661](https://github.com/opensearch-project/opensearch-py/pull/661))
 ### Deprecated
 ### Removed
 - Removed unnecessary `# -*- coding: utf-8 -*-` headers from .py files ([#615](https://github.com/opensearch-project/opensearch-py/pull/615), [#617](https://github.com/opensearch-project/opensearch-py/pull/617))
@@ -17,7 +23,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Fix reuse of `OpenSearch` using `Urllib3HttpConnection` and `AsyncOpenSearch` after calling `close` ([#639](https://github.com/opensearch-project/opensearch-py/pull/639))
 ### Security
 ### Dependencies
-- Bumps `pytest-asyncio` from <=0.21.1 to <=0.23.3
+- Bumps `pytest-asyncio` from <=0.21.1 to <=0.23.4
 - Bumps `urllib3` from >=1.26.18 to >=1.26.18, <2 ([#632](https://github.com/opensearch-project/opensearch-py/pull/632))
 
 ## [2.4.2]

benchmarks/bench_async.py

@@ -36,8 +36,9 @@ async def index_records(client: Any, index_name: str, item_count: int) -> None:
 
 async def test_async(client_count: int = 1, item_count: int = 1) -> None:
     """
-    asynchronously index with item_count records and run client_count clients. This function can be used to
-    test balancing the number of items indexed with the number of documents.
+    asynchronously index with item_count records and run client_count
+    clients. This function can be used to test balancing the number of
+    items indexed with the number of documents.
     """
     host = "localhost"
     port = 9200

benchmarks/bench_info_sync.py

@@ -22,13 +22,12 @@
 
 def get_info(client: Any, request_count: int) -> float:
     """get info from client"""
-    tt: float = 0
-    for n in range(request_count):
+    total_time: float = 0
+    for _ in range(request_count):
         start = time.time() * 1000
         client.info()
-        total_time = time.time() * 1000 - start
-        tt += total_time
-    return tt
+        total_time += time.time() * 1000 - start
+    return total_time
 
 
 def test(thread_count: int = 1, request_count: int = 1, client_count: int = 1) -> None:
@@ -50,7 +49,7 @@ def test(thread_count: int = 1, request_count: int = 1, client_count: int = 1) -
     root.addHandler(handler)
 
     clients = []
-    for i in range(client_count):
+    for _ in range(client_count):
         clients.append(
             OpenSearch(
                 hosts=[{"host": host, "port": port}],
@@ -71,8 +70,8 @@ def test(thread_count: int = 1, request_count: int = 1, client_count: int = 1) -
         thread.start()
 
     latency = 0
-    for t in threads:
-        latency += t.join()
+    for thread in threads:
+        latency += thread.join()
 
     print(f"latency={latency}")
 

benchmarks/bench_sync.py

@@ -23,29 +23,29 @@
 
 def index_records(client: Any, index_name: str, item_count: int) -> Any:
     """bulk index item_count records into index_name"""
-    tt = 0
-    for n in range(10):
+    total_time = 0
+    for _ in range(10):
         data: Any = []
-        for i in range(item_count):
+        for item in range(item_count):
             data.append(
                 json.dumps({"index": {"_index": index_name, "_id": str(uuid.uuid4())}})
             )
-            data.append(json.dumps({"value": i}))
+            data.append(json.dumps({"value": item}))
         data = "\n".join(data)
 
         start = time.time() * 1000
-        rc = client.bulk(data)
-        if rc["errors"]:
-            raise Exception(rc["errors"])
+        response = client.bulk(data)
+        if response["errors"]:
+            raise Exception(response["errors"])
 
-        server_time = rc["took"]
-        total_time = time.time() * 1000 - start
+        server_time = response["took"]
+        this_time = time.time() * 1000 - start
 
-        if total_time < server_time:
-            raise Exception(f"total={total_time} < server={server_time}")
+        if this_time < server_time:
+            raise Exception(f"total={this_time} < server={server_time}")
 
-        tt += total_time - server_time
-    return tt
+        total_time += this_time - server_time
+    return total_time
 
 
 def test(thread_count: int = 1, item_count: int = 1, client_count: int = 1) -> None:
@@ -68,7 +68,7 @@ def test(thread_count: int = 1, item_count: int = 1, client_count: int = 1) -> N
     root.addHandler(handler)
 
     clients = []
-    for i in range(client_count):
+    for _ in range(client_count):
         clients.append(
             OpenSearch(
                 hosts=[{"host": host, "port": port}],
@@ -105,8 +105,8 @@ def test(thread_count: int = 1, item_count: int = 1, client_count: int = 1) -> N
         thread.start()
 
     latency = 0
-    for t in threads:
-        latency += t.join()
+    for thread in threads:
+        latency += thread.join()
 
     clients[0].indices.refresh(index=index_name)
     count = clients[0].count(index=index_name)

dev-requirements.txt

@@ -21,5 +21,5 @@ twine
 
 # Requirements for testing [async] extra
 aiohttp
-pytest-asyncio<=0.23.3
+pytest-asyncio<=0.23.4
 unasync

guides/point_in_time.md

@@ -1,6 +1,7 @@
-- [Point-in-Time](#point-in-time)
+- [Point-in-Time - Deprecated Implementation Examples](#point-in-time---deprecated-implementation-examples)
+- [Point-in-Time - Updated Implementation Examples](#point-in-time---updated-implementation-examples)
 
-### Point-in-Time
+### Point-in-Time - Deprecated Implementation Examples
 
 [Point in Time (PIT)](https://opensearch.org/docs/latest/search-plugins/point-in-time/) lets you run different queries against a dataset that is fixed in time.
 
@@ -41,3 +42,43 @@ Delete all point in time.
 response = client.delete_point_in_time(body=None, all=True)
 print(response)
 ```
+
+
+### Point-in-Time - Updated Implementation Examples
+
+Create a point in time on an index.
+
+```python
+index_name = "test-index"
+response = client.create_pit(
+    index=index_name,
+    params={"keep_alive": "1m"}
+)
+
+pit_id = response.get("pit_id")
+print('\n Point in time ID:')
+print(pit_id)
+```
+
+
+List all point in time which are alive in the cluster.
+
+```python
+response = client.get_all_pits()
+print(response)
+```
+
+Delete a point in time.
+
+```python
+pit_body = {"pit_id": pit_id}
+response = client.delete_pit(body=pit_body)
+print(response)
+```
+
+Delete all point in time.
+
+```python
+response = client.delete_all_pits()
+print(response)
+```

noxfile.py

@@ -43,6 +43,10 @@
 
 @nox.session(python=["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"])  # type: ignore
 def test(session: Any) -> None:
+    """
+    runs all tests with a fresh python environment using "python setup.py test"
+    :param session: current nox session
+    """
     session.install(".")
     # ensure client can be imported without aiohttp
     session.run("python", "-c", "import opensearchpy\nprint(opensearchpy.OpenSearch())")
@@ -59,18 +63,26 @@ def test(session: Any) -> None:
 
 @nox.session(python=["3.7"])  # type: ignore
 def format(session: Any) -> None:
+    """
+    runs black and isort to format the files accordingly
+    :param session: current nox session
+    """
     session.install(".")
     session.install("black", "isort")
 
     session.run("isort", *SOURCE_FILES)
     session.run("black", *SOURCE_FILES)
     session.run("python", "utils/license_headers.py", "fix", *SOURCE_FILES)
 
-    lint(session)
+    session.notify("lint")
 
 
 @nox.session(python=["3.7"])  # type: ignore
 def lint(session: Any) -> None:
+    """
+    runs isort, black, flake8, pylint, and mypy to check the files according to each utility's function
+    :param session: current nox session
+    """
     session.install(
         "flake8",
         "black",
@@ -89,7 +101,13 @@ def lint(session: Any) -> None:
     session.run("isort", "--check", *SOURCE_FILES)
     session.run("black", "--check", *SOURCE_FILES)
     session.run("flake8", *SOURCE_FILES)
-    session.run("pylint", *SOURCE_FILES)
+
+    pylint_overrides = ["opensearchpy/", "test_opensearchpy/"]
+    pylint_defaults = [file for file in SOURCE_FILES if file not in pylint_overrides]
+    for file in pylint_overrides:
+        session.run("pylint", "--rcfile", f"{file}.pylintrc", f"{file}")
+    session.run("pylint", "--rcfile", ".pylintrc", *pylint_defaults)
+
     session.run("python", "utils/license_headers.py", "check", *SOURCE_FILES)
 
     # Workaround to make '-r' to still work despite uninstalling aiohttp below.
@@ -110,6 +128,10 @@ def lint(session: Any) -> None:
 
 @nox.session()  # type: ignore
 def docs(session: Any) -> None:
+    """
+    builds the html documentation for the client
+    :param session: current nox session
+    """
     session.install(".")
     session.install(".[docs]")
     with session.chdir("docs"):
@@ -118,6 +140,10 @@ def docs(session: Any) -> None:
 
 @nox.session()  # type: ignore
 def generate(session: Any) -> None:
+    """
+    generates the base API code
+    :param session: current nox session
+    """
     session.install("-rdev-requirements.txt")
     session.run("python", "utils/generate_api.py")
-    format(session)
+    session.notify("format")

opensearchpy/.pylintrc

@@ -0,0 +1,11 @@
+[MESSAGES CONTROL]
+disable=all
+enable=line-too-long,
+    invalid-name,
+    pointless-statement,
+    unspecified-encoding,
+    unnecessary-dunder-call,
+    assignment-from-no-return,
+    unused-variable
+max-line-length=240
+good-names-rgxs=^[_a-z][_a-z0-9]?$

opensearchpy/_async/client/__init__.py

@@ -1610,6 +1610,7 @@ async def scroll(
         "rest_total_hits_as_int",
         "routing",
         "scroll",
+        "search_pipeline",
         "search_type",
         "seq_no_primary_term",
         "size",
@@ -1704,6 +1705,8 @@ async def search(
         :arg routing: Comma-separated list of specific routing values.
         :arg scroll: Specify how long a consistent view of the index
             should be maintained for scrolled search.
+        :arg search_pipeline: Customizable sequence of processing stages
+            applied to search queries.
         :arg search_type: Search operation type. Valid choices are
             query_then_fetch, dfs_query_then_fetch.
         :arg seq_no_primary_term: Specify whether to return sequence