Master to staging (#1543)

* Hotfix db improvements (#1523) (#1524) * add extra configurations for postgresql connections * add master and slave replication strategy for typeorm Co-authored-by: CarlosQ96 <92376054+CarlosQ96@users.noreply.github.com> * Fix/db replica production (#1525) * Hotfix db improvements (#1523) * add extra configurations for postgresql connections * add master and slave replication strategy for typeorm * Define db read only configs --------- Co-authored-by: CarlosQ96 <92376054+CarlosQ96@users.noreply.github.com> * update comment * Hotfix latency issues for prod (#1529) * Hotfix staging fix latency (#1528) * add project donation summary view entity * convert projectQueries to querybuilder * add cache to projectDonationSummary queries * add configurable cache to slow queries * remove massive recurring donation log * add await for project queries * Add informative logs for draft donation service job (#1537) * Fix eslint errors * Fix/master test (#1541) * Fixed master test issue * Returned test to master pipeline * Comment executing donation summary view --------- Co-authored-by: Mohammad Ranjbar Z <mranjbar.z2993@gmail.com> --------- Co-authored-by: CarlosQ96 <92376054+CarlosQ96@users.noreply.github.com> Co-authored-by: Carlos <carlos.quintero096@gmail.com> Co-authored-by: mohammadranjbarz <mranjbar.z2993@gmail.com>
Giveth · May 7, 2024 · 7467f9b · 7467f9b
1 parent 8d1d38f
commit 7467f9b
Show file tree

Hide file tree

Showing 7 changed files with 73 additions and 36 deletions.
diff --git a/.github/workflows/master-pipeline.yml b/.github/workflows/master-pipeline.yml
@@ -89,26 +89,26 @@ jobs:
       - name: Run migrations
         run: npm run db:migrate:run:test
 
-#      - name: Run tests
-#        run: npm run test
-#        env:
-#          ETHERSCAN_API_KEY: ${{ secrets.ETHERSCAN_API_KEY }}
-#          XDAI_NODE_HTTP_URL: ${{ secrets.XDAI_NODE_HTTP_URL }}
-#          INFURA_API_KEY: ${{ secrets.INFURA_API_KEY }}
-#          INFURA_ID: ${{ secrets.INFURA_ID }}
-#          POLYGON_SCAN_API_KEY: ${{ secrets.POLYGON_SCAN_API_KEY }}
-#          OPTIMISTIC_SCAN_API_KEY: ${{ secrets.OPTIMISTIC_SCAN_API_KEY }}
-#          CELO_SCAN_API_KEY: ${{ secrets.CELO_SCAN_API_KEY }}
-#          CELO_ALFAJORES_SCAN_API_KEY: ${{ secrets.CELO_ALFAJORES_SCAN_API_KEY }}
-#          ARBITRUM_SCAN_API_KEY: ${{ secrets.ARBITRUM_SCAN_API_KEY }}
-#          ARBITRUM_SEPOLIA_SCAN_API_KEY: ${{ secrets.ARBITRUM_SEPOLIA_SCAN_API_KEY }}
-#          MORDOR_ETC_TESTNET: ${{ secrets.MORDOR_ETC_TESTNET }}
-#          ETC_NODE_HTTP_URL: ${{ secrets.ETC_NODE_HTTP_URL }}
-#          DROP_DATABASE: ${{ secrets.DROP_DATABASE_DURING_TEST_PROD }}
-#          SOLANA_TEST_NODE_RPC_URL: ${{ secrets.SOLANA_TEST_NODE_RPC_URL }}
-#          SOLANA_DEVNET_NODE_RPC_URL: ${{ secrets.SOLANA_DEVNET_NODE_RPC_URL }}
-#          SOLANA_MAINNET_NODE_RPC_URL: ${{ secrets.SOLANA_MAINNET_NODE_RPC_URL }}
-#          MPETH_GRAPHQL_PRICES_URL: ${{ secrets.MPETH_GRAPHQL_PRICES_URL }}
+      - name: Run tests
+        run: npm run test
+        env:
+          ETHERSCAN_API_KEY: ${{ secrets.ETHERSCAN_API_KEY }}
+          XDAI_NODE_HTTP_URL: ${{ secrets.XDAI_NODE_HTTP_URL }}
+          INFURA_API_KEY: ${{ secrets.INFURA_API_KEY }}
+          INFURA_ID: ${{ secrets.INFURA_ID }}
+          POLYGON_SCAN_API_KEY: ${{ secrets.POLYGON_SCAN_API_KEY }}
+          OPTIMISTIC_SCAN_API_KEY: ${{ secrets.OPTIMISTIC_SCAN_API_KEY }}
+          CELO_SCAN_API_KEY: ${{ secrets.CELO_SCAN_API_KEY }}
+          CELO_ALFAJORES_SCAN_API_KEY: ${{ secrets.CELO_ALFAJORES_SCAN_API_KEY }}
+          ARBITRUM_SCAN_API_KEY: ${{ secrets.ARBITRUM_SCAN_API_KEY }}
+          ARBITRUM_SEPOLIA_SCAN_API_KEY: ${{ secrets.ARBITRUM_SEPOLIA_SCAN_API_KEY }}
+          MORDOR_ETC_TESTNET: ${{ secrets.MORDOR_ETC_TESTNET }}
+          ETC_NODE_HTTP_URL: ${{ secrets.ETC_NODE_HTTP_URL }}
+          DROP_DATABASE: ${{ secrets.DROP_DATABASE_DURING_TEST_PROD }}
+          SOLANA_TEST_NODE_RPC_URL: ${{ secrets.SOLANA_TEST_NODE_RPC_URL }}
+          SOLANA_DEVNET_NODE_RPC_URL: ${{ secrets.SOLANA_DEVNET_NODE_RPC_URL }}
+          SOLANA_MAINNET_NODE_RPC_URL: ${{ secrets.SOLANA_MAINNET_NODE_RPC_URL }}
+          MPETH_GRAPHQL_PRICES_URL: ${{ secrets.MPETH_GRAPHQL_PRICES_URL }}
 
   publish:
     needs: test

diff --git a/src/entities/entities.ts b/src/entities/entities.ts
@@ -50,7 +50,7 @@ import { ProjectFraud } from './projectFraud';
 import { ProjectActualMatchingView } from './ProjectActualMatchingView';
 import { ProjectSocialMedia } from './projectSocialMedia';
 import { DraftRecurringDonation } from './draftRecurringDonation';
-import { ProjectDonationSummaryView } from './projectDonationSummaryView';
+import { ProjectDonationSummaryView } from '../views/projectDonationSummaryView';
 
 export const getEntities = (): DataSourceOptions['entities'] => {
   return [

diff --git a/src/orm.ts b/src/orm.ts
@@ -1,4 +1,5 @@
 import { DataSource } from 'typeorm';
+import { PostgresConnectionCredentialsOptions } from 'typeorm/driver/postgres/PostgresConnectionCredentialsOptions';
 import config from './config';
 import { CronJob } from './entities/CronJob';
 import { getEntities } from './entities/entities';
@@ -14,6 +15,16 @@ export class AppDataSource {
       const synchronize = (config.get('ENVIRONMENT') as string) === 'test';
       const entities = getEntities();
       const poolSize = Number(process.env.TYPEORM_DATABASE_POOL_SIZE) || 10; // 10 is the default value
+      const slaves: PostgresConnectionCredentialsOptions[] = [];
+      if (config.get('TYPEORM_DATABASE_HOST_READONLY')) {
+        slaves.push({
+          database: config.get('TYPEORM_DATABASE_NAME_READONLY') as string,
+          username: config.get('TYPEORM_DATABASE_USER_READONLY') as string,
+          password: config.get('TYPEORM_DATABASE_PASSWORD_READONLY') as string,
+          port: config.get('TYPEORM_DATABASE_PORT_READONLY') as number,
+          host: config.get('TYPEORM_DATABASE_HOST_READONLY') as string,
+        });
+      }
       AppDataSource.datasource = new DataSource({
         schema: 'public',
         type: 'postgres',
@@ -25,15 +36,7 @@ export class AppDataSource {
             port: config.get('TYPEORM_DATABASE_PORT') as number,
             host: config.get('TYPEORM_DATABASE_HOST') as string,
           },
-          slaves: [
-            {
-              database: config.get('TYPEORM_DATABASE_NAME') as string,
-              username: config.get('TYPEORM_DATABASE_USER') as string,
-              password: config.get('TYPEORM_DATABASE_PASSWORD') as string,
-              port: config.get('TYPEORM_DATABASE_PORT') as number,
-              host: config.get('TYPEORM_DATABASE_HOST_READONLY') as string,
-            },
-          ],
+          slaves,
         },
 
         entities,

diff --git a/src/repositories/donationRepository.ts b/src/repositories/donationRepository.ts
@@ -5,7 +5,7 @@ import { Donation, DONATION_STATUS } from '../entities/donation';
 import { ResourcesTotalPerMonthAndYear } from '../resolvers/donationResolver';
 import { getProjectDonationsSqrtRootSum } from './qfRoundRepository';
 import { logger } from '../utils/logger';
-import { ProjectDonationSummaryView } from '../entities/projectDonationSummaryView';
+import { ProjectDonationSummaryView } from '../views/projectDonationSummaryView';
 import { ProjectEstimatedMatchingView } from '../entities/ProjectEstimatedMatchingView';
 
 export const fillQfRoundDonationsUserScores = async (): Promise<void> => {

diff --git a/src/services/cronJobs/draftDonationMatchingJob.ts b/src/services/cronJobs/draftDonationMatchingJob.ts
@@ -30,9 +30,18 @@ export const runDraftDonationMatchWorkerJob = () => {
   });
 
   setInterval(async () => {
-    const count = await DraftDonation.countBy({
-      status: DRAFT_DONATION_STATUS.PENDING,
-    });
-    logger.debug('Pending Draft Donations count:', { count });
+    try {
+      logger.debug(
+        'Pending Draft Donations count: before execute the count query',
+      );
+      const count = await DraftDonation.count({
+        where: {
+          status: DRAFT_DONATION_STATUS.PENDING,
+        },
+      });
+      logger.debug('Pending Draft Donations count:', { count });
+    } catch (e) {
+      logger.error('Pending Draft Donations count: Error', e);
+    }
   }, TWO_MINUTES);
 };
diff --git a/src/services/donationService.ts b/src/services/donationService.ts
@@ -360,7 +360,7 @@ export const syncDonationStatusWithBlockchainNetwork = async (params: {
     // After updating, recalculate user total donated and owner total received
     await updateUserTotalDonated(donation.userId);
 
-    // After updating price we update totalDonations
+    // After updating price we update the totalDonations
     await updateTotalDonationsOfProject(donation.projectId);
     const project = await findProjectById(donation.projectId);
     await updateUserTotalReceived(project!.adminUser.id);

diff --git a/src/views/projectDonationSummaryView.ts b/src/views/projectDonationSummaryView.ts
@@ -0,0 +1,25 @@
+import { Field, ObjectType } from 'type-graphql';
+import {
+  Column,
+  PrimaryColumn,
+  BaseEntity,
+  ViewEntity,
+  ViewColumn,
+} from 'typeorm';
+
+@ViewEntity('project_donation_summary_view', { synchronize: false })
+@ObjectType()
+export class ProjectDonationSummaryView extends BaseEntity {
+  @Field()
+  @ViewColumn()
+  @PrimaryColumn()
+  projectId: number;
+
+  @ViewColumn()
+  @Column('double precision')
+  sumVerifiedDonations: number;
+
+  @ViewColumn()
+  @Column('int')
+  uniqueDonorsCount: number;
+}