Refactor/search (#16605)

* refactors Conan 2 search code

* refactor passing tests

* fix test

* Update conan/internal/cache/cache.py

Co-authored-by: Abril Rincón Blanco <git@rinconblanco.es>

---------

Co-authored-by: Abril Rincón Blanco <git@rinconblanco.es>

conan/api/subapi/cache.py

@@ -9,7 +9,7 @@
 from conan.internal.cache.cache import PkgCache
 from conan.internal.cache.home_paths import HomePaths
 from conan.internal.conan_app import ConanApp
-from conan.internal.integrity_check import IntegrityChecker
+from conan.internal.cache.integrity_check import IntegrityChecker
 from conans.client.downloaders.download_cache import DownloadCache
 from conans.errors import ConanException
 from conans.model.package_ref import PkgReference

conan/api/subapi/list.py

@@ -1,14 +1,18 @@
+import os
+from collections import OrderedDict
 from typing import Dict
 
 from conan.api.model import PackagesList
 from conan.api.output import ConanOutput, TimedOutput
+from conan.internal.api.list.query_parse import filter_package_configs
 from conan.internal.conan_app import ConanApp
+from conan.internal.paths import CONANINFO
 from conans.errors import ConanException, NotFoundException
 from conans.model.info import load_binary_info
 from conans.model.package_ref import PkgReference
 from conans.model.recipe_ref import RecipeReference, ref_matches
-from conans.search.search import get_cache_packages_binary_info, filter_packages
 from conans.util.dates import timelimit
+from conans.util.files import load
 
 
 class ListAPI:
@@ -69,7 +73,7 @@ def packages_configurations(self, ref: RecipeReference,
         if not remote:
             app = ConanApp(self.conan_api)
             prefs = app.cache.get_package_references(ref)
-            packages = get_cache_packages_binary_info(app.cache, prefs)
+            packages = _get_cache_packages_binary_info(app.cache, prefs)
         else:
             app = ConanApp(self.conan_api)
             if ref.revision == "latest":
@@ -85,7 +89,18 @@ def filter_packages_configurations(pkg_configurations, query):
         :param query: str like "os=Windows AND (arch=x86 OR compiler=gcc)"
         :return: Dict[PkgReference, PkgConfiguration]
         """
-        return filter_packages(query, pkg_configurations)
+        if query is None:
+            return pkg_configurations
+        try:
+            if "!" in query:
+                raise ConanException("'!' character is not allowed")
+            if "~" in query:
+                raise ConanException("'~' character is not allowed")
+            if " not " in query or query.startswith("not "):
+                raise ConanException("'not' operator is not allowed")
+            return filter_package_configs(pkg_configurations, query)
+        except Exception as exc:
+            raise ConanException("Invalid package query: %s. %s" % (query, exc))
 
     @staticmethod
     def filter_packages_profile(packages, profile, ref):
@@ -361,3 +376,29 @@ def serialize(self):
                 "python_requires": self.python_requires_diff,
                 "confs": self.confs_diff,
                 "explanation": self.explanation()}
+
+
+def _get_cache_packages_binary_info(cache, prefs) -> Dict[PkgReference, dict]:
+    """
+    param package_layout: Layout for the given reference
+    """
+
+    result = OrderedDict()
+
+    for pref in prefs:
+        latest_prev = cache.get_latest_package_reference(pref)
+        pkg_layout = cache.pkg_layout(latest_prev)
+
+        # Read conaninfo
+        info_path = os.path.join(pkg_layout.package(), CONANINFO)
+        if not os.path.exists(info_path):
+            raise ConanException(f"Corrupted package '{pkg_layout.reference}' "
+                                 f"without conaninfo.txt in: {info_path}")
+        conan_info_content = load(info_path)
+        info = load_binary_info(conan_info_content)
+        pref = pkg_layout.reference
+        # The key shoudln't have the latest package revision, we are asking for package configs
+        pref.revision = None
+        result[pkg_layout.reference] = info
+
+    return result

conan/api/subapi/search.py

@@ -1,5 +1,4 @@
 from conan.internal.conan_app import ConanApp
-from conans.search.search import search_recipes
 
 
 class SearchAPI:
@@ -17,7 +16,7 @@ def recipes(self, query: str, remote=None):
         if remote:
             refs = app.remote_manager.search_recipes(remote, query)
         else:
-            references = search_recipes(app.cache, query)
+            references = app.cache.search_recipes(query)
             # For consistency with the remote search, we return references without revisions
             # user could use further the API to look for the revisions
             refs = []

conans/search/query_parse.py → conan/internal/api/list/query_parse.py

@@ -1,14 +1,59 @@
+from collections import OrderedDict
 
-def is_operator(el):
+
+def filter_package_configs(pkg_configurations, query):
+    postfix = _infix_to_postfix(query) if query else []
+    result = OrderedDict()
+    for pref, data in pkg_configurations.items():
+        if _evaluate_postfix_with_info(postfix, data):
+            result[pref] = data
+    return result
+
+
+def _evaluate_postfix_with_info(postfix, binary_info):
+
+    # Evaluate conaninfo with the expression
+
+    def evaluate_info(expression):
+        """Receives an expression like compiler.version="12"
+        Uses conan_vars_info in the closure to evaluate it"""
+        name, value = expression.split("=", 1)
+        value = value.replace("\"", "")
+        return _evaluate(name, value, binary_info)
+
+    return _evaluate_postfix(postfix, evaluate_info)
+
+
+def _evaluate(prop_name, prop_value, binary_info):
+    """
+    Evaluates a single prop_name, prop_value like "os", "Windows" against
+    conan_vars_info.serialize_min()
+    """
+
+    def compatible_prop(setting_value, _prop_value):
+        return (_prop_value == setting_value) or (_prop_value == "None" and setting_value is None)
+
+    # TODO: Necessary to generalize this query evaluation to include all possible fields
+    info_settings = binary_info.get("settings", {})
+    info_options = binary_info.get("options", {})
+
+    if not prop_name.startswith("options."):
+        return compatible_prop(info_settings.get(prop_name), prop_value)
+    else:
+        prop_name = prop_name[len("options."):]
+        return compatible_prop(info_options.get(prop_name), prop_value)
+
+
+def _is_operator(el):
     return el in ["|", "&"]
 
 
 def _parse_expression(subexp):
-    '''Expressions like:
+    """Expressions like:
      compiler.version=12
      compiler="Visual Studio"
      arch="x86"
-     Could be replaced with another one to parse different queries'''
+     Could be replaced with another one to parse different queries """
     ret = ""
     quoted = False
     for char in subexp:
@@ -19,7 +64,7 @@ def _parse_expression(subexp):
 
         if quoted:
             ret += char
-        elif char == " " or is_operator(char) or char in [")", "("]:
+        elif char == " " or _is_operator(char) or char in [")", "("]:
             break
         else:
             ret += char
@@ -30,7 +75,7 @@ def _parse_expression(subexp):
     return ret
 
 
-def evaluate_postfix(postfix, evaluator):
+def _evaluate_postfix(postfix, evaluator):
     """
     Evaluates a postfix expression and returns a boolean
     @param postfix:  Postfix expression as a list
@@ -43,7 +88,7 @@ def evaluate_postfix(postfix, evaluator):
 
     stack = []
     for el in postfix:
-        if not is_operator(el):
+        if not _is_operator(el):
             stack.append(el)
         else:
             o1 = stack.pop()
@@ -66,7 +111,7 @@ def evaluate_postfix(postfix, evaluator):
         return stack[0]
 
 
-def infix_to_postfix(exp):
+def _infix_to_postfix(exp):
     """
     Translates an infix expression to postfix using an standard algorithm
     with little hacks for parse complex expressions like "compiler.version=4"
@@ -98,11 +143,11 @@ def infix_to_postfix(exp):
                     output.append(popped)
             if popped != "(":
                 raise Exception("Bad expression, not balanced parenthesis")
-        elif is_operator(char):
+        elif _is_operator(char):
             # Same operations has the same priority
             # replace this lines if the operators need to have
             # some priority
-            if stack and is_operator(stack[:-1]):
+            if stack and _is_operator(stack[:-1]):
                 popped = stack.pop()
                 output.append(popped)
             stack.append(char)

conan/internal/cache/cache.py

@@ -1,7 +1,9 @@
 import hashlib
 import os
+import re
 import shutil
 import uuid
+from fnmatch import translate
 from typing import List
 
 from conan.internal.cache.conan_reference_layout import RecipeLayout, PackageLayout
@@ -168,8 +170,18 @@ def update_recipe_timestamp(self, ref: RecipeReference):
         assert ref.timestamp
         self._db.update_recipe_timestamp(ref)
 
-    def all_refs(self):
-        return self._db.list_references()
+    def search_recipes(self, pattern=None, ignorecase=True):
+        # Conan references in main storage
+        if pattern:
+            if isinstance(pattern, RecipeReference):
+                pattern = repr(pattern)
+            pattern = translate(pattern)
+            pattern = re.compile(pattern, re.IGNORECASE if ignorecase else 0)
+
+        refs = self._db.list_references()
+        if pattern:
+            refs = [r for r in refs if r.partial_match(pattern)]
+        return refs
 
     def exists_prev(self, pref):
         # Used just by download to skip downloads if prev already exists in cache

conans/client/graph/range_resolver.py

@@ -2,7 +2,6 @@
 from conans.errors import ConanException
 from conans.model.recipe_ref import RecipeReference
 from conans.model.version_range import VersionRange
-from conans.search.search import search_recipes
 
 
 class RangeResolver:
@@ -57,7 +56,7 @@ def _resolve_local(self, search_ref, version_range):
         local_found = self._cached_cache.get(pattern)
         if local_found is None:
             # This local_found is weird, it contains multiple revisions, not just latest
-            local_found = search_recipes(self._cache, pattern)
+            local_found = self._cache.search_recipes(pattern)
             # TODO: This is still necessary to filter user/channel, until search_recipes is fixed
             local_found = [ref for ref in local_found if ref.user == search_ref.user
                            and ref.channel == search_ref.channel]

conans/model/recipe_ref.py

@@ -187,6 +187,23 @@ def matches(self, pattern, is_consumer):
             return not condition
         return condition
 
+    def partial_match(self, pattern):
+        """
+        Finds if pattern matches any of partial sums of tokens of conan reference
+        """
+        tokens = [self.name, "/", str(self.version)]
+        if self.user:
+            tokens += ["@", self.user]
+        if self.channel:
+            tokens += ["/", self.channel]
+        if self.revision:
+            tokens += ["#", self.revision]
+        partial = ""
+        for token in tokens:
+            partial += token
+            if pattern.match(partial):
+                return True
+
 
 def ref_matches(ref, pattern, is_consumer):
     if not ref or not str(ref):