[server] Create columns for product details

Create separate columns for products to store the number of runs and
the latest storage date instead of connecting and getting these
information from each product database.

web/server/codechecker_server/api/mass_store_run.py

@@ -18,7 +18,7 @@
 from datetime import datetime
 from hashlib import sha256
 from tempfile import TemporaryDirectory
-from typing import Any, Dict, List, NamedTuple, Optional, Set
+from typing import Any, Dict, List, NamedTuple, Optional, Set, Tuple
 
 import codechecker_api_shared
 from codechecker_api.codeCheckerDBAccess_v6 import ttypes
@@ -676,7 +676,7 @@ def __add_checker_run(
         self,
         session: DBSession,
         run_history_time: datetime
-    ) -> int:
+    ) -> Tuple[int, bool]:
         """
         Store run related data to the database.
         By default updates the results if name already exists.
@@ -689,6 +689,7 @@ def __add_checker_run(
                 .filter(Run.name == self.__name) \
                 .one_or_none()
 
+            update_run = True
             if run and self.__force:
                 # Clean already collected results.
                 if not run.can_delete:
@@ -723,6 +724,7 @@ def __add_checker_run(
                 session.add(checker_run)
                 session.flush()
                 run_id = checker_run.id
+                update_run = False
 
             # Add run to the history.
             LOG.debug("Adding run history.")
@@ -758,7 +760,7 @@ def __add_checker_run(
             session.flush()
             LOG.debug("Storing analysis statistics done.")
 
-            return run_id
+            return run_id, update_run
         except Exception as ex:
             raise codechecker_api_shared.ttypes.RequestFailed(
                 codechecker_api_shared.ttypes.ErrorCode.GENERAL,
@@ -1189,7 +1191,7 @@ def store(self) -> int:
                                       self.__name, run_lock.locked_at)
 
                             # Actual store operation begins here.
-                            run_id = self.__add_checker_run(
+                            run_id, update_run = self.__add_checker_run(
                                 session, run_history_time)
 
                             LOG.info("[%s] Store reports...", self.__name)
@@ -1202,14 +1204,24 @@ def store(self) -> int:
 
                             session.commit()
 
-                            LOG.info("'%s' stored results (%s KB "
-                                     "/decompressed/) to run '%s' (id: %d) in "
-                                     "%s seconds.", self.user_name,
-                                     round(zip_size / 1024),
-                                     self.__name, run_id,
-                                     round(time.time() - start_time, 2))
+                        inc_num_of_runs = 1
+
+                        # If it's a run update, do not increment the number
+                        # of runs of the current product.
+                        if update_run:
+                            inc_num_of_runs = None
+
+                        self.__report_server._set_run_data_for_curr_product(
+                            inc_num_of_runs, run_history_time)
+
+                        LOG.info("'%s' stored results (%s KB "
+                                 "/decompressed/) to run '%s' (id: %d) in "
+                                 "%s seconds.", self.user_name,
+                                 round(zip_size / 1024),
+                                 self.__name, run_id,
+                                 round(time.time() - start_time, 2))
 
-                            return run_id
+                        return run_id
                     except (sqlalchemy.exc.OperationalError,
                             sqlalchemy.exc.ProgrammingError) as ex:
                         num_of_tries += 1

web/server/codechecker_server/api/product_server.py

@@ -88,26 +88,6 @@ def __get_product(self, session, product):
             self.__server.add_product(product)
             server_product = self.__server.get_product(product.endpoint)
 
-        num_of_runs = 0
-        runs_in_progress = set()
-        latest_store_to_product = ""
-
-        # Try to get product details. It may throw different error messages
-        # depending on the used SQL driver adapter in case of connection error,
-        # so we should try to connect to the database and get the results
-        # again on failure.
-        try:
-            num_of_runs, runs_in_progress, latest_store_to_product = \
-                server_product.get_details()
-        except Exception:
-            # Try to connect to the product and try to get details again.
-            server_product.connect()
-
-            if server_product.db_status ==\
-                    codechecker_api_shared.ttypes.DBStatus.OK:
-                num_of_runs, runs_in_progress, latest_store_to_product = \
-                    server_product.get_details()
-
         descr = convert.to_b64(product.description) \
             if product.description else None
 
@@ -132,14 +112,14 @@ def __get_product(self, session, product):
             endpoint=product.endpoint,
             displayedName_b64=convert.to_b64(product.display_name),
             description_b64=descr,
-            runCount=num_of_runs,
-            latestStoreToProduct=str(latest_store_to_product),
+            runCount=product.num_of_runs,
+            latestStoreToProduct=str(product.latest_storage_date)
+                if product.latest_storage_date else None,
             connected=connected,
             accessible=product_access,
             administrating=product_admin,
             databaseStatus=server_product.db_status,
-            admins=[admin.name for admin in admins],
-            runStoreInProgress=runs_in_progress)
+            admins=[admin.name for admin in admins])
 
     @timeit
     def getPackageVersion(self):

web/server/codechecker_server/api/report_server.py

@@ -17,7 +17,7 @@
 
 from collections import defaultdict
 from datetime import datetime, timedelta
-from typing import List
+from typing import List, Optional
 
 import sqlalchemy
 from sqlalchemy.sql.expression import or_, and_, not_, func, \
@@ -985,6 +985,30 @@ def _get_username(self):
         """
         return self._auth_session.user if self._auth_session else "Anonymous"
 
+    def _set_run_data_for_curr_product(
+        self,
+        inc_num_of_runs: Optional[int],
+        latest_storage_date: Optional[datetime] = None
+    ):
+        """
+        Increment the number of runs related to the current product with the
+        given value and set the latest storage date.
+        """
+        values = {}
+
+        if inc_num_of_runs is not None:
+            values["num_of_runs"] = Product.num_of_runs + inc_num_of_runs
+
+        if latest_storage_date is not None:
+            values["latest_storage_date"] = latest_storage_date
+
+        with DBSession(self._config_database) as session:
+            session.query(Product) \
+                .filter(Product.id == self._product.id) \
+                .update(values)
+
+            session.commit()
+
     def __require_permission(self, required):
         """
         Helper method to raise an UNAUTHORIZED exception if the user does not
@@ -2515,6 +2539,10 @@ def removeRun(self, run_id, run_filter):
             LOG.info("Runs '%s' were removed by '%s'.", runs,
                      self._get_username())
 
+        # Decrement the number of runs but do not update the latest storage
+        # date.
+        self._set_run_data_for_curr_product(-1)
+
         # Remove unused data (files, comments, etc.) from the database.
         db_cleanup.remove_unused_data(self._Session)
 

web/server/codechecker_server/database/config_db_model.py

@@ -39,6 +39,8 @@ class Product(Base):
     display_name = Column(String, nullable=False)
     description = Column(Text)
     run_limit = Column(Integer)
+    num_of_runs = Column(Integer, server_default="0")
+    latest_storage_date = Column(DateTime, nullable=True)
 
     # Disable review status change on UI.
     is_review_status_change_disabled = Column(Boolean,

web/server/codechecker_server/migrations/config/env.py

@@ -17,6 +17,8 @@
 config = context.config
 
 # Add model's MetaData object here for 'autogenerate' support.
+sys.path.insert(0, os.path.abspath(
+    os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "..")))
 sys.path.insert(0, os.path.abspath(
     os.path.join(os.path.dirname(__file__), "..", "..", "..")))
 sys.path.insert(0, os.path.abspath(

web/server/codechecker_server/migrations/config/versions/4db450cf38af_add_extra_product_detail_columns.py

@@ -0,0 +1,71 @@
+"""Add columns for number of runs and latest run storage
+
+Revision ID: 4db450cf38af
+Revises: 302693c76eb8
+Create Date: 2021-06-28 15:52:57.237509
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '4db450cf38af'
+down_revision = '302693c76eb8'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+
+from codechecker_common.logger import get_logger
+from codechecker_server.database import database
+from codechecker_server.database.run_db_model import IDENTIFIER as RUN_META
+from codechecker_web.shared import webserver_context
+
+LOG = get_logger('system')
+
+
+def upgrade():
+    op.add_column(
+        'products',
+        sa.Column(
+            'latest_storage_date',
+            sa.DateTime(),
+            nullable=True))
+
+    op.add_column(
+        'products',
+        sa.Column(
+            'num_of_runs',
+            sa.Integer(),
+            server_default="0",
+            nullable=True))
+
+    try:
+        product_con = op.get_bind()
+        products = product_con.execute(
+            "SELECT id, connection from products").fetchall()
+
+        context = webserver_context.get_context()
+        for id, connection in products:
+            sql_server = database.SQLServer.from_connection_string(
+                connection, RUN_META, context.run_migration_root)
+
+            engine = sa.create_engine(sql_server.get_connection_string())
+            conn = engine.connect()
+
+            run_info = \
+                conn.execute("SELECT count(*), max(date) from runs").fetchone()
+
+            product_con.execute(f"""
+                UPDATE products
+                SET num_of_runs={run_info[0]},
+                    latest_storage_date='{run_info[1]}'
+                WHERE id={id}
+            """)
+    except:
+        LOG.error("Failed to fill product detail columns (num_of_runs, "
+                  "latest_storage_date)!")
+        pass
+
+def downgrade():
+    op.drop_column('products', 'num_of_runs')
+    op.drop_column('products', 'latest_storage_date')

web/server/vue-cli/src/views/Products.vue

@@ -107,24 +107,6 @@
         </v-chip>
       </template>
 
-      <template #item.runStoreInProgress="{ item }">
-        <span
-          v-for="runName in item.runStoreInProgress"
-          :key="runName"
-          class="v-chip-max-width-wrapper"
-        >
-          <v-chip
-            class="mr-2 my-1"
-            color="accent"
-          >
-            <v-avatar>
-              <v-icon>mdi-play-circle</v-icon>
-            </v-avatar>
-            {{ runName }}
-          </v-chip>
-        </span>
-      </template>
-
       <template v-slot:item.action="{ item }">
         <div class="text-no-wrap">
           <edit-product-btn
@@ -205,11 +187,6 @@ export default {
           value: "latestStoreToProduct",
           sortable: true
         },
-        {
-          text: "Run store in progress",
-          value: "runStoreInProgress",
-          sortable: false
-        },
         {
           text: "Actions",
           value: "action",

web/tests/functional/store/test_store.py

@@ -60,10 +60,48 @@ def setUp(self):
         self._temp_workspace = os.path.join(self._codechecker_cfg["workspace"],
                                             "test_proj")
 
+        self.product_name = self._codechecker_cfg['viewer_product']
+
         # Setup a viewer client to test viewer API calls.
         self._cc_client = env.setup_viewer_client(self._test_workspace)
         self.assertIsNotNone(self._cc_client)
 
+        self._pr_client = env.setup_product_client(
+            self._test_workspace, product=self.product_name)
+        self.assertIsNotNone(self._pr_client)
+
+    def test_product_details(self):
+        """
+        Test that product details columns are set properly on run
+        storage / removal events.
+        """
+        product = self._pr_client.getCurrentProduct()
+        self.assertFalse(product.runCount)
+        self.assertFalse(product.latestStoreToProduct)
+
+        run_name = "product_details_test"
+        store_cmd = [
+            env.codechecker_cmd(), "store", self._temp_workspace,
+            "--name", run_name,
+            "--url", env.parts_to_url(self._codechecker_cfg)]
+
+        _call_cmd(store_cmd)
+
+        product = self._pr_client.getCurrentProduct()
+        self.assertTrue(product.runCount)
+        self.assertTrue(product.latestStoreToProduct)
+
+        rm_cmd = [
+            env.codechecker_cmd(), "cmd", "del",
+            "-n", run_name,
+            "--url", env.parts_to_url(self._codechecker_cfg)]
+
+        _call_cmd(rm_cmd)
+
+        product = self._pr_client.getCurrentProduct()
+        self.assertFalse(product.runCount)
+        self.assertTrue(product.latestStoreToProduct)
+
     def test_trim_path_prefix_store(self):
         """Trim the path prefix from the sored reports.
 
@@ -249,7 +287,7 @@ def store_multiple_report_dirs(report_dirs):
 
         shutil.rmtree(common_report_dir, ignore_errors=True)
 
-    def test_duplicated_suppress(self):
+    def test_suppress_duplicated(self):
         """
         Test server if recognise duplicated suppress comments in the stored
         source code.