Merge branch 'port-labs:main' into clickup-integrations

CHANGELOG.md

@@ -7,6 +7,14 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 <!-- towncrier release notes start -->
 
+## 0.10.0 (2024-08-19)
+
+### Improvements
+
+- Add support for reporting the integration resync state to expose more information about the integration state in the portal
+- Fix kafka listener never ending resync loop due to resyncState updates
+
+
 ## 0.9.14 (2024-08-19)
 
 

integrations/azure-devops/.port/resources/blueprints.json

@@ -115,5 +115,91 @@
         "many": false
       }
     }
+  },
+  {
+    "identifier": "workItem",
+    "title": "Work Item",
+    "icon": "AzureDevops",
+    "schema": {
+      "properties": {
+        "type": {
+          "title": "Type",
+          "type": "string",
+          "icon": "AzureDevops",
+          "description": "The type of work item (e.g., Bug, Task, User Story)",
+          "enum": ["Issue", "Epic", "Task"],
+          "enumColors": {
+            "Issue": "green",
+            "Epic": "orange",
+            "Task": "blue"
+          }
+        },
+        "state": {
+          "title": "State",
+          "type": "string",
+          "icon": "AzureDevops",
+          "description": "The current state of the work item (e.g., New, Active, Closed)"
+        },
+        "reason": {
+          "title": "Reason",
+          "type": "string",
+          "description": "The title of the work item"
+        },
+        "effort": {
+          "title": "Effort",
+          "type": "number",
+          "description": "The estimated effort for the work item"
+        },
+        "description": {
+          "title": "Description",
+          "type": "string",
+          "format": "markdown",
+          "description": "A detailed description of the work item"
+        },
+        "link": {
+          "title": "Link",
+          "type": "string",
+          "format": "url",
+          "icon": "AzureDevops",
+          "description": "Link to the work item in Azure DevOps"
+        },
+        "createdBy": {
+          "title": "Created By",
+          "type": "string",
+          "icon": "User",
+          "description": "The person who created the work item"
+        },
+        "changedBy": {
+          "title": "Changed By",
+          "type": "string",
+          "icon": "User",
+          "description": "The person who last changed the work item"
+        },
+        "createdDate": {
+          "title": "Created Date",
+          "type": "string",
+          "format": "date-time",
+          "description": "The date and time when the work item was created"
+        },
+        "changedDate": {
+          "title": "Changed Date",
+          "type": "string",
+          "format": "date-time",
+          "description": "The date and time when the work item was last changed"
+        }
+      },
+      "required": []
+    },
+    "mirrorProperties": {},
+    "calculationProperties": {},
+    "aggregationProperties": {},
+    "relations": {
+      "project": {
+        "title": "Project",
+        "target": "project",
+        "required": true,
+        "many": false
+      }
+    }
   }
 ]

integrations/azure-devops/.port/resources/port-app-config.yaml

@@ -17,7 +17,6 @@ resources:
             visibility: '.visibility'
             defaultTeam: '.defaultTeam.name'
             link: '.url | gsub("_apis/projects/"; "")'
-
   - kind: repository
     selector:
       query: "true"
@@ -54,3 +53,25 @@ resources:
           blueprint: '"service"'
           properties:
             workItemLinking: '.isEnabled and .isBlocking'
+  - kind: work-item
+    selector:
+      query: 'true'
+    port:
+      entity:
+        mappings:
+          identifier: '.id | tostring'
+          blueprint: '"workItem"'
+          title: '.fields."System.Title"'
+          properties:
+            type: '.fields."System.WorkItemType"'
+            state: '.fields."System.State"'
+            effort: '.fields."Microsoft.VSTS.Scheduling.Effort"'
+            description: '.fields."System.Description"'
+            link: '.url'
+            reason: '.fields."System.Reason"'
+            createdBy: '.fields."System.CreatedBy".displayName'
+            changedBy: '.fields."System.ChangedBy".displayName'
+            createdDate: '.fields."System.CreatedDate"'
+            changedDate: '.fields."System.ChangedDate"'
+          relations:
+            project: '.fields."System.TeamProject" | gsub(" "; "")'

integrations/azure-devops/CHANGELOG.md

@@ -6,6 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 <!-- towncrier release notes start -->
+## 0.1.54 (2024-08-21)
+
+### Features
+
+- Added work items to get issues, tasks, and epics
 
 ## 0.1.53 (2024-08-20)
 

integrations/azure-devops/azure_devops/client/azure_devops_client.py

@@ -1,16 +1,26 @@
 import json
+import asyncio
+import typing
+
 from typing import Any, AsyncGenerator, Optional
-from azure_devops.webhooks.webhook_event import WebhookEvent
 from httpx import HTTPStatusError
+from loguru import logger
+
 from port_ocean.context.event import event
 from port_ocean.context.ocean import ocean
-from loguru import logger
-from .base_client import HTTPBaseClient
 from port_ocean.utils.cache import cache_iterator_result
-import asyncio
+
+from azure_devops.misc import AzureDevopsWorkItemResourceConfig
+from azure_devops.webhooks.webhook_event import WebhookEvent
+
+from .base_client import HTTPBaseClient
+
 
 API_URL_PREFIX = "_apis"
 WEBHOOK_API_PARAMS = {"api-version": "7.1-preview.1"}
+# Maximum number of work item IDs allowed in a single API request
+# (based on Azure DevOps API limitations) https://learn.microsoft.com/en-us/rest/api/azure/devops/wit/work-items/list?view=azure-devops-rest-7.1&tabs=HTTP
+MAX_WORK_ITEMS_PER_REQUEST = 200
 
 
 class AzureDevopsClient(HTTPBaseClient):
@@ -137,6 +147,91 @@ async def generate_repository_policies(
                     policy["__repository"] = repo
                 yield repo_policies
 
+    async def generate_work_items(self) -> AsyncGenerator[list[dict[str, Any]], None]:
+        """
+        Retrieves a paginated list of work items within the Azure DevOps organization based on a WIQL query.
+        """
+        async for projects in self.generate_projects():
+            for project in projects:
+                # 1. Execute WIQL query to get work item IDs
+                work_item_ids = await self._fetch_work_item_ids(project["id"])
+
+                # 2. Fetch work items using the IDs (in batches if needed)
+                work_items = await self._fetch_work_items_in_batches(
+                    project["id"], work_item_ids
+                )
+
+                # Call the private method to add __projectId to each work item
+                self._add_project_id_to_work_items(work_items, project["id"])
+
+                yield work_items
+
+    async def _fetch_work_item_ids(self, project_id: str) -> list[int]:
+        """
+        Executes a WIQL query to fetch work item IDs for a given project.
+
+        :param project_id: The ID of the project.
+        :return: A list of work item IDs.
+        """
+        config = typing.cast(AzureDevopsWorkItemResourceConfig, event.resource_config)
+
+        wiql_query = "SELECT [Id] from WorkItems"
+
+        if config.selector.wiql:
+            # Append the user-provided wiql to the WHERE clause
+            wiql_query += f" WHERE {config.selector.wiql}"
+            logger.info(f"Found and appended WIQL filter: {config.selector.wiql}")
+
+        wiql_url = (
+            f"{self._organization_base_url}/{project_id}/{API_URL_PREFIX}/wit/wiql"
+        )
+        wiql_response = await self.send_request(
+            "POST",
+            wiql_url,
+            params={"api-version": "7.1-preview.2"},
+            data=json.dumps({"query": wiql_query}),
+            headers={"Content-Type": "application/json"},
+        )
+        wiql_response.raise_for_status()
+        return [item["id"] for item in wiql_response.json()["workItems"]]
+
+    async def _fetch_work_items_in_batches(
+        self, project_id: str, work_item_ids: list[int]
+    ) -> list[dict[str, Any]]:
+        """
+        Fetches work items in batches based on the list of work item IDs.
+
+        :param project_id: The ID of the project.
+        :param work_item_ids: A list of work item IDs to fetch.
+        :return: A list of work items.
+        """
+        work_items = []
+        for i in range(0, len(work_item_ids), MAX_WORK_ITEMS_PER_REQUEST):
+            batch_ids = work_item_ids[i : i + MAX_WORK_ITEMS_PER_REQUEST]
+            work_items_url = f"{self._organization_base_url}/{project_id}/{API_URL_PREFIX}/wit/workitems"
+            params = {
+                "ids": ",".join(map(str, batch_ids)),
+                "api-version": "7.1-preview.3",
+            }
+            work_items_response = await self.send_request(
+                "GET", work_items_url, params=params
+            )
+            work_items_response.raise_for_status()
+            work_items.extend(work_items_response.json()["value"])
+        return work_items
+
+    def _add_project_id_to_work_items(
+        self, work_items: list[dict[str, Any]], project_id: str
+    ) -> None:
+        """
+        Adds the project ID to each work item in the list.
+
+        :param work_items: List of work items to modify.
+        :param project_id: The project ID to add to each work item.
+        """
+        for work_item in work_items:
+            work_item["__projectId"] = project_id
+
     async def get_pull_request(self, pull_request_id: str) -> dict[Any, Any]:
         get_single_pull_request_url = f"{self._organization_base_url}/{API_URL_PREFIX}/git/pullrequests/{pull_request_id}"
         response = await self.send_request("GET", get_single_pull_request_url)

integrations/azure-devops/azure_devops/misc.py

@@ -18,6 +18,7 @@ class Kind(StrEnum):
     MEMBER = "member"
     TEAM = "team"
     PROJECT = "project"
+    WORK_ITEM = "work-item"
 
 
 PULL_REQUEST_SEARCH_CRITERIA: list[dict[str, Any]] = [
@@ -46,12 +47,27 @@ class AzureDevopsSelector(Selector):
     selector: AzureDevopsSelector
 
 
+class AzureDevopsWorkItemResourceConfig(ResourceConfig):
+    class AzureDevopsSelector(Selector):
+        query: str
+        wiql: str | None = Field(
+            default=None,
+            description="WIQL query to filter work items. If not provided, all work items will be fetched.",
+            alias="wiql",
+        )
+
+    kind: Literal["work-item"]
+    selector: AzureDevopsSelector
+
+
 class GitPortAppConfig(PortAppConfig):
     spec_path: List[str] | str = Field(alias="specPath", default="port.yml")
     branch: str = "main"
-    resources: list[AzureDevopsProjectResourceConfig | ResourceConfig] = Field(
-        default_factory=list
-    )
+    resources: list[
+        AzureDevopsProjectResourceConfig
+        | AzureDevopsWorkItemResourceConfig
+        | ResourceConfig
+    ] = Field(default_factory=list)
 
 
 def extract_branch_name_from_ref(ref: str) -> str:

integrations/azure-devops/main.py

@@ -103,6 +103,14 @@ async def resync_repository_policies(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE:
         yield policies
 
 
+@ocean.on_resync(Kind.WORK_ITEM)
+async def resync_workitems(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE:
+    azure_devops_client = AzureDevopsClient.create_from_ocean_config()
+    async for work_items in azure_devops_client.generate_work_items():
+        logger.info(f"Resyncing {len(work_items)} work items")
+        yield work_items
+
+
 @ocean.router.post("/webhook")
 async def webhook(request: Request) -> dict[str, Any]:
     body = await request.json()

integrations/azure-devops/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "azure-devops"
-version = "0.1.53"
+version = "0.1.54"
 description = "An Azure Devops Ocean integration"
 authors = ["Matan Geva <matang@getport.io>"]
 

port_ocean/clients/port/client.py

@@ -13,6 +13,7 @@
     get_internal_http_client,
 )
 from port_ocean.exceptions.clients import KafkaCredentialsNotFound
+from typing import Any
 
 
 class PortClient(
@@ -75,3 +76,19 @@ async def get_org_id(self) -> str:
         handle_status_code(response)
 
         return response.json()["organization"]["id"]
+
+    async def update_integration_state(
+        self, state: dict[str, Any], should_raise: bool = True, should_log: bool = True
+    ) -> dict[str, Any]:
+        if should_log:
+            logger.debug(f"Updating integration resync state with: {state}")
+        response = await self.client.patch(
+            f"{self.api_url}/integration/{self.integration_identifier}/resync-state",
+            headers=await self.auth.headers(),
+            json=state,
+        )
+        handle_status_code(response, should_raise, should_log)
+        if response.is_success and should_log:
+            logger.info("Integration resync state updated successfully")
+
+        return response.json().get("integration", {})

port_ocean/config/settings.py

@@ -76,7 +76,7 @@ class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
     integration: IntegrationSettings = Field(
         default_factory=lambda: IntegrationSettings(type="", identifier="")
     )
-    runtime: Runtime = "OnPrem"
+    runtime: Runtime = Runtime.OnPrem
 
     @root_validator()
     def validate_integration_config(cls, values: dict[str, Any]) -> dict[str, Any]:
@@ -100,8 +100,8 @@ def parse_config(model: Type[BaseModel], config: Any) -> BaseModel:
         return values
 
     @validator("runtime")
-    def validate_runtime(cls, runtime: Literal["OnPrem", "Saas"]) -> Runtime:
-        if runtime == "Saas":
+    def validate_runtime(cls, runtime: Runtime) -> Runtime:
+        if runtime == Runtime.Saas:
             spec = get_spec_file()
             if spec is None:
                 raise ValueError(