Add pre-release information to exported data (#3841)

* Add pre-release information to exported data

* reset if not needed

* Fix data content

custom_components/hacs/repositories/base.py

@@ -112,6 +112,7 @@
     ("last_commit", None),
     ("last_updated", 0),
     ("last_version", None),
+    ("prerelease", None),
     ("manifest_name", None),
     ("open_issues", 0),
     ("stargazers_count", 0),
@@ -166,6 +167,7 @@ class RepositoryData:
     new: bool = True
     open_issues: int = 0
     published_tags: list[str] = []
+    prerelease: str = None
     releases: bool = False
     selected_tag: str = None
     show_beta: bool = False

custom_components/hacs/utils/data.py

@@ -46,6 +46,7 @@
     ("last_commit", None),
     ("last_version", None),
     ("manifest_name", None),
+    ("prerelease", None),
     ("open_issues", 0),
     ("published_tags", []),
     ("releases", False),
@@ -290,6 +291,7 @@ def async_restore_repository(self, entry: str, repository_data: dict[str, Any]):
         repository.data.show_beta = repository_data.get("show_beta", False)
         repository.data.last_version = repository_data.get("last_version")
         repository.data.last_commit = repository_data.get("last_commit")
+        repository.data.prerelease = repository_data.get("prerelease")
         repository.data.installed_version = repository_data.get("version_installed")
         repository.data.installed_commit = repository_data.get("installed_commit")
         repository.data.manifest_name = repository_data.get("manifest_name")
@@ -308,6 +310,9 @@ def async_restore_repository(self, entry: str, repository_data: dict[str, Any]):
         if repository.data.installed:
             repository.data.first_install = False
 
+        if repository.data.prerelease == repository.data.last_version:
+            repository.data.prerelease = None
+
         if entry == HACS_REPOSITORY_ID:
             repository.data.installed_version = self.hacs.version
             repository.data.installed = True

custom_components/hacs/utils/validate.py

@@ -118,6 +118,7 @@ def validate_version(data: Any) -> Any:
     vol.Required("last_fetched"): vol.Any(int, float),
     vol.Required("last_updated"): str,
     vol.Optional("last_version"): str,
+    vol.Optional("prerelease"): str,
     vol.Required("manifest"): {
         vol.Optional("country"): vol.Any([str], False),
         vol.Optional("name"): str,

scripts/data/generate_category_data.py

@@ -130,9 +130,7 @@ def async_store_repository_data(self, repository: HacsRepository) -> dict:
         """Store the repository data."""
         data = {"manifest": {}}
         for key, default in HACS_MANIFEST_KEYS_TO_EXPORT:
-            if (
-                value := getattr(repository.repository_manifest, key, default)
-            ) != default:
+            if (value := getattr(repository.repository_manifest, key, default)) != default:
                 data["manifest"][key] = value
 
         for key, default in REPOSITORY_KEYS_TO_EXPORT:
@@ -166,8 +164,7 @@ def __init__(self, session: ClientSession, *, token: str | None = None):
         self.core.config_path = None
         self.configuration.token = token
         self.data = AdjustedHacsData(hacs=self)
-        self.data_client = HacsDataClient(
-            session=session, client_name="HACS/Generator")
+        self.data_client = HacsDataClient(session=session, client_name="HACS/Generator")
 
         self.github = GitHub(
             token,
@@ -205,31 +202,51 @@ async def concurrent_update_repository(self, repository: HacsRepository) -> None
             repository.data.etag_repository = None
 
         if repository.data.last_version not in (None, ""):
+            releases: list[GitHubReleaseModel] = []
             try:
                 repository.logger.info(
                     "%s Fetching repository releases",
                     repository.string,
                 )
                 response = await self.githubapi.generic(
-                    endpoint=f"/repos/{repository.data.full_name}/releases/latest",
+                    endpoint=f"/repos/{repository.data.full_name}/releases",
                     etag=repository.data.etag_releases,
+                    kwargs={"per_page": 1},
                 )
-                response.data = (
-                    GitHubReleaseModel(
-                        response.data) if response.data else None
-                )
+
+                releases = [GitHubReleaseModel(rel) for rel in response.data]
                 repository.data.etag_releases = response.etag
-                if (releases := response.data) is not None:
-                    repository.data.releases = True
-                    repository.releases.objects = [releases]
-                    repository.data.published_tags = [
-                        x.tag_name for x in repository.releases.objects
-                    ]
-                    if (
-                        next_version := next(iter(repository.data.published_tags), None)
-                    ) != repository.data.last_version:
-                        repository.data.last_version = next_version
-                        repository.data.etag_repository = None
+
+                if len(releases) != 0:
+                    first = releases[0]
+                    if first.prerelease:
+                        repository.logger.info("%s Found prerelease", repository.string)
+                        repository.data.prerelease = first.tag_name
+                        response = await self.githubapi.generic(
+                            endpoint=f"/repos/{repository.data.full_name}/releases/latest",
+                            etag=repository.data.etag_releases,
+                        )
+                        if content := response.data:
+                            releases = [GitHubReleaseModel(content)]
+                        else:
+                            releases = []
+                    elif repository.data.prerelease is not None:
+                        repository.data.prerelease = None
+
+                    if len(releases) != 0:
+                        repository.data.releases = True
+                        repository.releases.objects = releases
+                        repository.data.published_tags = [
+                            x.tag_name for x in repository.releases.objects
+                        ]
+                        if (
+                            next_version := next(iter(repository.data.published_tags), None)
+                        ) != repository.data.last_version:
+                            repository.data.last_version = next_version
+                            repository.data.etag_repository = None
+
+                    if repository.data.prerelease == repository.data.last_version:
+                        repository.data.prerelease = None
 
             except GitHubNotModifiedException:
                 repository.data.releases = True
@@ -239,12 +256,10 @@ async def concurrent_update_repository(self, repository: HacsRepository) -> None
                 )
             except GitHubNotFoundException:
                 repository.data.releases = False
-                repository.logger.info(
-                    "%s No releases found", repository.string)
+                repository.logger.info("%s No releases found", repository.string)
             except GitHubException as exception:
                 repository.data.releases = False
-                repository.logger.warning(
-                    "%s %s", repository.string, exception)
+                repository.logger.warning("%s %s", repository.string, exception)
 
         await repository.common_update(
             force=repository.data.etag_repository is None,
@@ -330,8 +345,7 @@ async def get_category_repositories(
                 continue
             repository = self.repositories.get_by_full_name(repo)
             if repository is not None:
-                self.queue.add(self.concurrent_update_repository(
-                    repository=repository))
+                self.queue.add(self.concurrent_update_repository(repository=repository))
                 continue
 
             self.queue.add(
@@ -409,8 +423,7 @@ async def async_github_get_hacs_default_file(self, filename: str) -> list:
 async def generate_category_data(category: str, repository_name: str = None):
     """Generate data."""
     async with ClientSession() as session:
-        hacs = AdjustedHacs(
-            session=session, token=os.getenv("DATA_GENERATOR_TOKEN"))
+        hacs = AdjustedHacs(session=session, token=os.getenv("DATA_GENERATOR_TOKEN"))
         os.makedirs(os.path.join(OUTPUT_DIR, category), exist_ok=True)
         os.makedirs(os.path.join(OUTPUT_DIR, "diff"), exist_ok=True)
         force = os.environ.get("FORCE_REPOSITORY_UPDATE") == "True"
@@ -450,11 +463,7 @@ async def generate_category_data(category: str, repository_name: str = None):
             )
 
         did_raise = False
-        if (
-            not updated_data
-            or len(updated_data) == 0
-            or not isinstance(updated_data, dict)
-        ):
+        if not updated_data or len(updated_data) == 0 or not isinstance(updated_data, dict):
             print_error_and_exit(f"Updated data is empty", category)
             did_raise = True
 
@@ -471,8 +480,7 @@ async def generate_category_data(category: str, repository_name: str = None):
             print_error_and_exit(f"Invalid data: {errors}", category)
 
         if did_raise:
-            print_error_and_exit(
-                "Validation did raise but did not exit!", category)
+            print_error_and_exit("Validation did raise but did not exit!", category)
             sys.exit(1)  # Fallback, should not be reached
 
         with open(

tests/fixtures/proxy/api.github.com/repos/hacs-test-org/integration-basic/releases.json

@@ -1,4 +1,78 @@
 [
+    {
+        "url": "https://api.github.com/repos/hacs-test-org/integration-basic/releases/1",
+        "html_url": "https://github.com/hacs-test-org/integration-basic/releases/2.0.0b0",
+        "assets_url": "https://api.github.com/repos/hacs-test-org/integration-basic/releases/1/assets",
+        "upload_url": "https://uploads.github.com/repos/hacs-test-org/integration-basic/releases/1/assets{?name,label}",
+        "tarball_url": "https://api.github.com/repos/hacs-test-org/integration-basic/tarball/2.0.0b0",
+        "zipball_url": "https://api.github.com/repos/hacs-test-org/integration-basic/zipball/2.0.0b0",
+        "id": 1,
+        "node_id": "MDc6UmVsZWFzZTE=",
+        "tag_name": "2.0.0b0",
+        "target_commitish": "master",
+        "name": "2.0.0b0",
+        "body": "Description of the release",
+        "draft": false,
+        "prerelease": true,
+        "created_at": "2013-02-27T19:35:32Z",
+        "published_at": "2013-02-27T19:35:32Z",
+        "author": {
+            "login": "octocat",
+            "id": 1,
+            "node_id": "MDQ6VXNlcjE=",
+            "avatar_url": "https://github.com/images/error/octocat_happy.gif",
+            "gravatar_id": "",
+            "url": "https://api.github.com/users/octocat",
+            "html_url": "https://github.com/octocat",
+            "followers_url": "https://api.github.com/users/hacs-test-org/followers",
+            "following_url": "https://api.github.com/users/hacs-test-org/following{/other_user}",
+            "gists_url": "https://api.github.com/users/hacs-test-org/gists{/gist_id}",
+            "starred_url": "https://api.github.com/users/hacs-test-org/starred{/owner}{/repo}",
+            "subscriptions_url": "https://api.github.com/users/hacs-test-org/subscriptions",
+            "organizations_url": "https://api.github.com/users/hacs-test-org/orgs",
+            "repos_url": "https://api.github.com/users/hacs-test-org/repos",
+            "events_url": "https://api.github.com/users/hacs-test-org/events{/privacy}",
+            "received_events_url": "https://api.github.com/users/hacs-test-org/received_events",
+            "type": "User",
+            "site_admin": false
+        },
+        "assets": [
+            {
+                "url": "https://api.github.com/repos/hacs-test-org/integration-basic/releases/assets/1",
+                "browser_download_url": "https://github.com/hacs-test-org/integration-basic/releases/download/2.0.0b0/example.zip",
+                "id": 1,
+                "node_id": "MDEyOlJlbGVhc2VBc3NldDE=",
+                "name": "example.zip",
+                "label": "short description",
+                "state": "uploaded",
+                "content_type": "application/zip",
+                "size": 1024,
+                "download_count": 42,
+                "created_at": "2013-02-27T19:35:32Z",
+                "updated_at": "2013-02-27T19:35:32Z",
+                "uploader": {
+                    "login": "octocat",
+                    "id": 1,
+                    "node_id": "MDQ6VXNlcjE=",
+                    "avatar_url": "https://github.com/images/error/octocat_happy.gif",
+                    "gravatar_id": "",
+                    "url": "https://api.github.com/users/octocat",
+                    "html_url": "https://github.com/octocat",
+                    "followers_url": "https://api.github.com/users/hacs-test-org/followers",
+                    "following_url": "https://api.github.com/users/hacs-test-org/following{/other_user}",
+                    "gists_url": "https://api.github.com/users/hacs-test-org/gists{/gist_id}",
+                    "starred_url": "https://api.github.com/users/hacs-test-org/starred{/owner}{/repo}",
+                    "subscriptions_url": "https://api.github.com/users/hacs-test-org/subscriptions",
+                    "organizations_url": "https://api.github.com/users/hacs-test-org/orgs",
+                    "repos_url": "https://api.github.com/users/hacs-test-org/repos",
+                    "events_url": "https://api.github.com/users/hacs-test-org/events{/privacy}",
+                    "received_events_url": "https://api.github.com/users/hacs-test-org/received_events",
+                    "type": "User",
+                    "site_admin": false
+                }
+            }
+        ]
+    },
     {
         "url": "https://api.github.com/repos/hacs-test-org/integration-basic/releases/1",
         "html_url": "https://github.com/hacs-test-org/integration-basic/releases/1.0.0",

tests/fixtures/proxy/api.github.com/repos/hacs-test-org/integration-basic/releases/latest.json

@@ -0,0 +1,76 @@
+[
+    {
+        "url": "https://api.github.com/repos/hacs-test-org/integration-basic/releases/1",
+        "html_url": "https://github.com/hacs-test-org/integration-basic/releases/1.0.0",
+        "assets_url": "https://api.github.com/repos/hacs-test-org/integration-basic/releases/1/assets",
+        "upload_url": "https://uploads.github.com/repos/hacs-test-org/integration-basic/releases/1/assets{?name,label}",
+        "tarball_url": "https://api.github.com/repos/hacs-test-org/integration-basic/tarball/1.0.0",
+        "zipball_url": "https://api.github.com/repos/hacs-test-org/integration-basic/zipball/1.0.0",
+        "id": 1,
+        "node_id": "MDc6UmVsZWFzZTE=",
+        "tag_name": "1.0.0",
+        "target_commitish": "master",
+        "name": "1.0.0",
+        "body": "Description of the release",
+        "draft": false,
+        "prerelease": false,
+        "created_at": "2013-02-27T19:35:32Z",
+        "published_at": "2013-02-27T19:35:32Z",
+        "author": {
+            "login": "octocat",
+            "id": 1,
+            "node_id": "MDQ6VXNlcjE=",
+            "avatar_url": "https://github.com/images/error/octocat_happy.gif",
+            "gravatar_id": "",
+            "url": "https://api.github.com/users/octocat",
+            "html_url": "https://github.com/octocat",
+            "followers_url": "https://api.github.com/users/hacs-test-org/followers",
+            "following_url": "https://api.github.com/users/hacs-test-org/following{/other_user}",
+            "gists_url": "https://api.github.com/users/hacs-test-org/gists{/gist_id}",
+            "starred_url": "https://api.github.com/users/hacs-test-org/starred{/owner}{/repo}",
+            "subscriptions_url": "https://api.github.com/users/hacs-test-org/subscriptions",
+            "organizations_url": "https://api.github.com/users/hacs-test-org/orgs",
+            "repos_url": "https://api.github.com/users/hacs-test-org/repos",
+            "events_url": "https://api.github.com/users/hacs-test-org/events{/privacy}",
+            "received_events_url": "https://api.github.com/users/hacs-test-org/received_events",
+            "type": "User",
+            "site_admin": false
+        },
+        "assets": [
+            {
+                "url": "https://api.github.com/repos/hacs-test-org/integration-basic/releases/assets/1",
+                "browser_download_url": "https://github.com/hacs-test-org/integration-basic/releases/download/1.0.0/example.zip",
+                "id": 1,
+                "node_id": "MDEyOlJlbGVhc2VBc3NldDE=",
+                "name": "example.zip",
+                "label": "short description",
+                "state": "uploaded",
+                "content_type": "application/zip",
+                "size": 1024,
+                "download_count": 42,
+                "created_at": "2013-02-27T19:35:32Z",
+                "updated_at": "2013-02-27T19:35:32Z",
+                "uploader": {
+                    "login": "octocat",
+                    "id": 1,
+                    "node_id": "MDQ6VXNlcjE=",
+                    "avatar_url": "https://github.com/images/error/octocat_happy.gif",
+                    "gravatar_id": "",
+                    "url": "https://api.github.com/users/octocat",
+                    "html_url": "https://github.com/octocat",
+                    "followers_url": "https://api.github.com/users/hacs-test-org/followers",
+                    "following_url": "https://api.github.com/users/hacs-test-org/following{/other_user}",
+                    "gists_url": "https://api.github.com/users/hacs-test-org/gists{/gist_id}",
+                    "starred_url": "https://api.github.com/users/hacs-test-org/starred{/owner}{/repo}",
+                    "subscriptions_url": "https://api.github.com/users/hacs-test-org/subscriptions",
+                    "organizations_url": "https://api.github.com/users/hacs-test-org/orgs",
+                    "repos_url": "https://api.github.com/users/hacs-test-org/repos",
+                    "events_url": "https://api.github.com/users/hacs-test-org/events{/privacy}",
+                    "received_events_url": "https://api.github.com/users/hacs-test-org/received_events",
+                    "type": "User",
+                    "site_admin": false
+                }
+            }
+        ]
+    }
+]

tests/output/proxy_calls.json

@@ -1252,6 +1252,22 @@
         "https://api.github.com/repos/hacs-test-org/theme-basic/releases": 1,
         "https://data-v2.hacs.xyz/theme/data.json": 1
     },
+    "tests/scripts/data/test_generate_category_data.py::test_generate_category_data_with_prior_content[category_test_data0]": {
+        "https://api.github.com/rate_limit": 1,
+        "https://api.github.com/repos/hacs-test-org/integration-basic-custom": 1,
+        "https://api.github.com/repos/hacs-test-org/integration-basic-custom/contents/custom_components/example/manifest.json": 1,
+        "https://api.github.com/repos/hacs-test-org/integration-basic-custom/contents/hacs.json": 1,
+        "https://api.github.com/repos/hacs-test-org/integration-basic-custom/git/trees/1.0.0": 1,
+        "https://api.github.com/repos/hacs-test-org/integration-basic-custom/releases": 1,
+        "https://api.github.com/repos/hacs-test-org/integration-basic/releases": 1,
+        "https://api.github.com/repos/hacs-test-org/integration-basic/releases/latest": 1,
+        "https://api.github.com/repos/hacs/default/contents/integration": 1,
+        "https://api.github.com/repos/hacs/integration": 1,
+        "https://api.github.com/repos/hacs/integration/branches/main": 1,
+        "https://api.github.com/repos/hacs/integration/releases": 1,
+        "https://data-v2.hacs.xyz/integration/data.json": 1,
+        "https://data-v2.hacs.xyz/removed/repositories.json": 1
+    },
     "tests/test_config_flow.py::test_flow_with_activation_failure": {
         "https://github.com/login/device/code": 1,
         "https://github.com/login/oauth/access_token": 2