From cc4ee60d5dff694b34a97adc1837993add81b183 Mon Sep 17 00:00:00 2001 From: johnykes Date: Thu, 23 Mar 2023 17:50:07 +0200 Subject: [PATCH 01/18] SERVICES-1085 bulk operations + renaming --- src/db/auctions/auctions.repository.ts | 81 ++++++++++++++++- src/db/auctions/tags.repository.ts | 34 +++++++ src/db/offers/offers.repository.ts | 62 ++++++++++++- src/db/orders/orders.repository.ts | 91 +++++++++++++++++-- .../auctions/auctions-setter.service.ts | 8 +- .../notifications/notifications.service.ts | 2 +- src/modules/orders/order.service.ts | 12 +++ 7 files changed, 271 insertions(+), 19 deletions(-) diff --git a/src/db/auctions/auctions.repository.ts b/src/db/auctions/auctions.repository.ts index c179b0f42..0ac32043e 100644 --- a/src/db/auctions/auctions.repository.ts +++ b/src/db/auctions/auctions.repository.ts @@ -421,10 +421,83 @@ export class AuctionsRepository { return await this.auctionsRepository.save(auction); } - async saveBulkAuctions(auctions: AuctionEntity[]): Promise { - await this.auctionsRepository.save(auctions, { - chunk: constants.dbBatch, - }); + async getBulkAuctionsByMarketplaceAndAuctionIds( + marketplaceKey: string, + marketplaceAuctionIds: number[], + ): Promise { + return await this.auctionsRepository + .createQueryBuilder('a') + .select('id, marketplaceAuctionId') + .where(`a.marketplaceKey = '${marketplaceKey}'`) + .andWhere(`a.marketplaceAuctionId IN(:...marketplaceAuctionIds)`, { + marketplaceAuctionIds: marketplaceAuctionIds, + }) + .orderBy('a.marketplaceAuctionId', 'ASC') + .execute(); + } + + async saveBulkAuctionsOrUpdateAndFillId( + auctions: AuctionEntity[], + ): Promise { + if (auctions.length === 0) { + return; + } + const saveOrUpdateResponse = await this.auctionsRepository + .createQueryBuilder() + .insert() + .into('auctions') + .values(auctions) + .orUpdate({ + overwrite: [ + 'creationDate', + 'modifiedDate', + 'collection', + 'nrAuctionedTokens', + 'identifier', + 'nonce', + 'status', + 'type', + 'paymentToken', + 'paymentNonce', + 'ownerAddress', + 'minBidDiff', + 'minBid', + 'minBidDenominated', + 'maxBid', + 'maxBidDenominated', + 'startDate', + 'endDate', + 'tags', + 'blockHash', + ], + conflict_target: ['marketplaceAuctionId', 'marketplaceKey'], + }) + .updateEntity(false) + .execute(); + if ( + saveOrUpdateResponse.identifiers.length === 0 || + auctions.findIndex((a) => a.id === undefined) !== -1 + ) { + const dbAuctions = await this.getBulkAuctionsByMarketplaceAndAuctionIds( + auctions?.[0]?.marketplaceKey, + auctions?.map((a) => a.marketplaceAuctionId), + ); + for (let i = 0; i < dbAuctions.length; i++) { + const auctionIndex = auctions.findIndex( + (a) => a.marketplaceAuctionId === dbAuctions[i].marketplaceAuctionId, + ); + auctions[auctionIndex].id = dbAuctions[i].id; + } + } + if (auctions.findIndex((a) => a.id === undefined) !== -1) { + const wtf = auctions + .filter((a) => a.id === undefined) + .map((a) => a.marketplaceAuctionId); + const duplicate = auctions.filter( + (a) => a.marketplaceAuctionId === 31434, + ); + throw new Error(`oooppps ${JSON.stringify(duplicate)}`); + } } async rollbackAuctionAndOrdersByHash(blockHash: string): Promise { diff --git a/src/db/auctions/tags.repository.ts b/src/db/auctions/tags.repository.ts index 90648a62a..791c60755 100644 --- a/src/db/auctions/tags.repository.ts +++ b/src/db/auctions/tags.repository.ts @@ -81,4 +81,38 @@ export class TagsRepository { throw err; } } + + async getBulkTagsByAuctionIds(auctionIds: number[]): Promise { + return await this.tagsRepository + .createQueryBuilder('t') + .select('t.id, t.auctionId, t.tag') + .where(`t.auctionId IN(:...auctionIds)`, { + auctionIds: auctionIds, + }) + .execute(); + } + + async saveTagsOrIgnore(tags: TagEntity[]): Promise { + if (tags.length === 0) { + return; + } + const dbTags = await this.getBulkTagsByAuctionIds( + tags.map((t) => t.auctionId), + ); + for (const dbTag of dbTags) { + const correspondingTagIndex = tags.findIndex( + (t) => t.auctionId === dbTag.auctionId && t.tag === dbTag.tag, + ); + if (correspondingTagIndex !== -1) { + tags[correspondingTagIndex].id = dbTag.id; + } + } + await this.tagsRepository + .createQueryBuilder() + .insert() + .into('tags') + .values(tags.filter((t) => t.id === undefined)) + .orIgnore(true) + .execute(); + } } diff --git a/src/db/offers/offers.repository.ts b/src/db/offers/offers.repository.ts index 9f9262da6..a3fe7fe25 100644 --- a/src/db/offers/offers.repository.ts +++ b/src/db/offers/offers.repository.ts @@ -59,10 +59,64 @@ export class OffersRepository { return savedOffer; } - async saveBulkOffers(orders: OfferEntity[]): Promise { - await this.offersRepository.save(orders, { - chunk: constants.dbBatch, - }); + async getBulkIdsByMarketplaceAndOfferIds( + marketplaceKey: string, + marketplaceOfferIds: number[], + ): Promise { + const res = await this.offersRepository + .createQueryBuilder('o') + .select('id') + .where(`o.marketplaceKey = '${marketplaceKey}'`) + .andWhere(`o.marketplaceOfferId IN(:...marketplaceOfferIds)`, { + marketplaceOfferIds: marketplaceOfferIds, + }) + .orderBy('marketplaceOfferId', 'ASC') + .execute(); + return res.map((o) => o.id); + } + + async saveBulkOffersOrUpdateAndFillId(offers: OfferEntity[]): Promise { + if (offers.length === 0) { + return; + } + const saveOrUpdateResponse = await this.offersRepository + .createQueryBuilder() + .insert() + .into('offers') + .values(offers) + .orUpdate({ + overwrite: [ + 'creationDate', + 'modifiedDate', + 'boughtTokensNo', + 'status', + 'priceToken', + 'priceNonce', + 'priceAmount', + 'priceAmountDenominated', + 'ownerAddress', + 'endDate', + 'blockHash', + ], + conflict_target: ['marketplaceOfferId', 'marketplaceKey'], + }) + .updateEntity(false) + .execute(); + if ( + saveOrUpdateResponse.identifiers.length === 0 || + offers.findIndex((a) => a.id === undefined) !== -1 + ) { + const ids = await this.getBulkIdsByMarketplaceAndOfferIds( + offers?.[0]?.marketplaceKey, + offers?.map((o) => o.marketplaceOfferId), + ); + for (let i = 0; i < offers.length; i++) { + offers[i].id = ids[i]; + } + } + if (offers.findIndex((a) => a.id === undefined) !== -1) { + throw new Error('oooppps'); + } } async updateOfferWithStatus(offer: OfferEntity, status: OfferStatusEnum) { diff --git a/src/db/orders/orders.repository.ts b/src/db/orders/orders.repository.ts index de49f71d3..ffe428754 100644 --- a/src/db/orders/orders.repository.ts +++ b/src/db/orders/orders.repository.ts @@ -63,17 +63,25 @@ export class OrdersRepository { .getMany(); } - async getOrdersByAuctionIds(auctionIds: number[]): Promise { + async getOrdersByAuctionIdsGroupByAuctionId(auctionIds: number[]): Promise { const orders = await this.ordersRepository - .createQueryBuilder('orders') - .where(`auctionId IN(:...auctionIds) and status in ('active')`, { + .createQueryBuilder('o') + .where(`o.auctionId IN(:...auctionIds)`, { auctionIds: auctionIds, }) .getMany(); - return orders?.groupBy((asset) => asset.auctionId); } + async getOrdersByAuctionIds(auctionIds: number[]): Promise { + return await this.ordersRepository + .createQueryBuilder('o') + .where(`o.auctionId IN(:...auctionIds)`, { + auctionIds: auctionIds, + }) + .getMany(); + } + async getOrders(queryRequest: QueryRequest): Promise<[OrderEntity[], number]> { const filterQueryBuilder = new FilterQueryBuilder(this.ordersRepository, queryRequest.filters); const queryBuilder: SelectQueryBuilder = filterQueryBuilder.build(); @@ -88,10 +96,77 @@ export class OrdersRepository { return await this.ordersRepository.save(order); } - async saveBulkOrders(orders: OrderEntity[]): Promise { - await this.ordersRepository.save(orders, { - chunk: constants.dbBatch, - }); + async getBulkOrdersByMarketplaceAndAuctionIds(orders: OrderEntity[]): Promise { + const auctionIds = orders.filter((o) => o.id === undefined).map((o) => o.auctionId); + if (auctionIds.length === 0) { + return []; + } + const ordersResponse = await this.ordersRepository + .createQueryBuilder('o') + .select('*') + .where(`o.auctionId IN(:...auctionIds)`, { + auctionIds: auctionIds, + }) + .execute(); + + for (let i = 0; i < orders.length; i++) { + if (orders[i].id !== undefined) { + continue; + } + let similarOrders = orders.filter( + (o) => + o.auctionId === orders[i].auctionId && + o.ownerAddress === orders[i].ownerAddress && + o.priceToken === orders[i].priceToken && + o.priceAmount === orders[i].priceAmount, + ); + const similarOrdersWithIdsCount = similarOrders.filter((o) => o.id !== undefined).length; + const correspondingDbOrders = ordersResponse.filter( + (o: OrderEntity) => + o.auctionId === orders[i].auctionId && + o.ownerAddress === orders[i].ownerAddress && + o.priceToken === orders[i].priceToken && + o.priceAmount === orders[i].priceAmount, + ); + orders[i].id = correspondingDbOrders?.[similarOrders.length - similarOrdersWithIdsCount - 1]?.id; + } + return orders; + } + + async saveBulkOrdersOrUpdateAndFillId(orders: OrderEntity[]): Promise { + if (orders.length === 0) { + return; + } + orders = await this.getBulkOrdersByMarketplaceAndAuctionIds(orders); + + const saveOrUpdateResponse = await this.ordersRepository + .createQueryBuilder() + .insert() + .into('orders') + .values(orders) + .orUpdate({ + overwrite: [ + 'creationDate', + 'modifiedDate', + 'boughtTokensNo', + 'status', + 'priceToken', + 'priceNonce', + 'priceAmount', + 'priceAmountDenominated', + 'ownerAddress', + 'blockHash', + ], + conflict_target: ['id', 'auctionId', 'blockHash'], + }) + .execute(); + + if (saveOrUpdateResponse.identifiers.length === 0 || orders.findIndex((a) => a.id === undefined) !== -1) { + orders = await this.getBulkOrdersByMarketplaceAndAuctionIds(orders); + } + if (orders.findIndex((a) => a.id === undefined) !== -1) { + throw new Error('oooppps'); + } } async updateOrderWithStatus(order: OrderEntity, status: OrderStatusEnum) { diff --git a/src/modules/auctions/auctions-setter.service.ts b/src/modules/auctions/auctions-setter.service.ts index 6cb4f2706..81455b3a6 100644 --- a/src/modules/auctions/auctions-setter.service.ts +++ b/src/modules/auctions/auctions-setter.service.ts @@ -89,8 +89,12 @@ export class AuctionsSetterService { } } - async saveBulkAuctions(auctions: AuctionEntity[]): Promise { - return await this.persistenceService.saveBulkAuctions(auctions); + async saveBulkAuctionsOrUpdateAndFillId( + auctions: AuctionEntity[], + ): Promise { + return await this.persistenceService.saveBulkAuctionsOrUpdateAndFillId( + auctions, + ); } async saveAuctionEntity(auctionEntity: AuctionEntity, assetTags: string[]): Promise { diff --git a/src/modules/notifications/notifications.service.ts b/src/modules/notifications/notifications.service.ts index 7957299f9..1e9760709 100644 --- a/src/modules/notifications/notifications.service.ts +++ b/src/modules/notifications/notifications.service.ts @@ -37,7 +37,7 @@ export class NotificationsService { async generateNotifications(auctions: AuctionEntity[]): Promise { await this.updateNotificationStatus(auctions?.map((a) => a.id)); - const orders = await this.ordersService.getOrdersByAuctionIds(auctions?.map((a) => a.id)); + const orders = await this.ordersService.getOrdersByAuctionIdsGroupByAuctionId(auctions?.map((a) => a.id)); for (const auction of auctions) { this.addNotifications(auction, orders[auction.id]); } diff --git a/src/modules/orders/order.service.ts b/src/modules/orders/order.service.ts index 49dac9203..c33e2d1d8 100644 --- a/src/modules/orders/order.service.ts +++ b/src/modules/orders/order.service.ts @@ -146,6 +146,18 @@ export class OrdersService { return this.ordersCachingService.getOrSetOrders(queryRequest, () => this.getMappedOrders(queryRequest)); } + async getOrdersByAuctionIdsGroupByAuctionId( + auctionIds: number[], + ): Promise { + if (auctionIds?.length > 0) { + const orders = + await this.persistenceService.getOrdersByAuctionIdsGroupByAuctionId( + auctionIds, + ); + return orders; + } + } + async getOrdersByAuctionIds(auctionIds: number[]): Promise { if (auctionIds?.length > 0) { const orders = await this.persistenceService.getOrdersByAuctionIds(auctionIds); From 2d8ac951f4e5d410d5772c3e00e9a7d176a3d24e Mon Sep 17 00:00:00 2001 From: johnykes Date: Thu, 23 Mar 2023 17:51:19 +0200 Subject: [PATCH 02/18] SERVICES-1085 BigNumber.toFixed() and check for duplicates before creating auction/offer (for hybrid reindexing states) --- .../reindex-auction-started.handler.ts | 9 ++++++--- .../reindex-offer-created.hander.ts | 8 ++++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-started.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-started.handler.ts index a93351c5f..3f58fb6eb 100644 --- a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-started.handler.ts +++ b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-started.handler.ts @@ -1,5 +1,6 @@ import { BinaryUtils } from '@multiversx/sdk-nestjs-common'; import { Injectable } from '@nestjs/common'; +import BigNumber from 'bignumber.js'; import { constants } from 'src/config'; import { AuctionEntity } from 'src/db/auctions'; import { AuctionStatusEnum } from 'src/modules/auctions/models'; @@ -22,10 +23,12 @@ export class ReindexAuctionStartedHandler { const minBidDenominated = BigNumberUtils.denominateAmount(input.minBid, paymentToken.decimals); const maxBidDenominated = BigNumberUtils.denominateAmount(input.maxBid !== 'NaN' ? input.maxBid : '0', paymentToken.decimals); + marketplaceReindexState.deleteAuctionIfDuplicates(input.auctionId); + const auction = new AuctionEntity({ creationDate: modifiedDate, modifiedDate, - id: marketplaceReindexState.auctions.length, + id: marketplaceReindexState.getNewAuctionId(), marketplaceAuctionId: input.auctionId !== 0 ? input.auctionId : marketplaceReindexState.auctions.length + 1, identifier: input.identifier, collection: input.collection, @@ -36,8 +39,8 @@ export class ReindexAuctionStartedHandler { paymentToken: paymentToken.identifier, paymentNonce, ownerAddress: input.sender, - minBid: input.minBid, - maxBid: input.maxBid !== 'NaN' ? input.maxBid : '0', + minBid: new BigNumber(input.minBid).toFixed(), + maxBid: new BigNumber(input.maxBid !== 'NaN' ? input.maxBid : '0').toFixed(), minBidDenominated: Math.min(minBidDenominated, constants.dbMaxDenominatedValue), maxBidDenominated: Math.min(maxBidDenominated, constants.dbMaxDenominatedValue), minBidDiff: input.minBidDiff ?? '0', diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-offer-created.hander.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-offer-created.hander.ts index 7e45e29c6..d0faa5b76 100644 --- a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-offer-created.hander.ts +++ b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-offer-created.hander.ts @@ -1,4 +1,5 @@ import { Injectable } from '@nestjs/common'; +import BigNumber from 'bignumber.js'; import { constants } from 'src/config'; import { OfferEntity } from 'src/db/offers'; import { OfferStatusEnum } from 'src/modules/offers/models'; @@ -14,8 +15,11 @@ export class ReindexOfferCreatedHandler { handle(marketplaceReindexState: MarketplaceReindexState, input: OfferCreatedSummary, decimals: number): void { const modifiedDate = DateUtils.getUtcDateFromTimestamp(input.timestamp); const priceAmountDenominated = BigNumberUtils.denominateAmount(input.price, decimals); + + marketplaceReindexState.deleteOfferIfDuplicates(input.offerId); + const offer = new OfferEntity({ - id: marketplaceReindexState.offers.length, + id: marketplaceReindexState.getNewOfferId(), creationDate: modifiedDate, modifiedDate, marketplaceOfferId: input.offerId, @@ -24,7 +28,7 @@ export class ReindexOfferCreatedHandler { identifier: input.identifier, priceToken: input.paymentToken, priceNonce: input.paymentNonce, - priceAmount: input.price, + priceAmount: new BigNumber(input.price).toFixed(), priceAmountDenominated: Math.min(priceAmountDenominated, constants.dbMaxDenominatedValue), ownerAddress: input.address, endDate: Math.min(input.endTime, constants.dbMaxTimestamp), From 7bf7cf0164e4f2e1a6f1ca25c73667ff4145e60b Mon Sep 17 00:00:00 2001 From: johnykes Date: Thu, 23 Mar 2023 17:52:14 +0200 Subject: [PATCH 03/18] SERVICES-1085 hybrid/partial reindexing states (items in memory and DB) --- src/common/persistence/persistence.service.ts | 26 ++- src/config/default.json | 3 +- ...etplaces-reindex-events-summary.service.ts | 10 +- .../marketplaces-reindex.service.ts | 211 +++++++++--------- .../marketplaces/marketplaces.module.ts | 2 + .../models/MarketplaceReindexState.ts | 108 +++++++-- 6 files changed, 234 insertions(+), 126 deletions(-) diff --git a/src/common/persistence/persistence.service.ts b/src/common/persistence/persistence.service.ts index 64061323d..3ea8bfe86 100644 --- a/src/common/persistence/persistence.service.ts +++ b/src/common/persistence/persistence.service.ts @@ -153,6 +153,10 @@ export class PersistenceService { return await this.execute(this.saveTags.name, this.tagsRepository.saveTags(tags)); } + async saveTagsOrIgnore(tags: TagEntity[]): Promise { + await this.execute(this.saveTagsOrIgnore.name, this.tagsRepository.saveTagsOrIgnore(tags)); + } + async getCollectionStats( identifier: string, marketplaceKey: string = undefined, @@ -456,6 +460,13 @@ export class PersistenceService { return await this.execute(this.getLastOrdersByAuctionIds.name, this.ordersRepository.getLastOrdersByAuctionIds(auctionIds)); } + async getOrdersByAuctionIdsGroupByAuctionId(auctionIds: number[]): Promise { + return await this.execute( + this.getOrdersByAuctionIdsGroupByAuctionId.name, + this.ordersRepository.getOrdersByAuctionIdsGroupByAuctionId(auctionIds), + ); + } + async getOrdersByAuctionIds(auctionIds: number[]): Promise { return await this.execute(this.getOrdersByAuctionIds.name, this.ordersRepository.getOrdersByAuctionIds(auctionIds)); } @@ -468,8 +479,8 @@ export class PersistenceService { return await this.execute(this.saveOrder.name, this.ordersRepository.saveOrder(order)); } - async saveBulkOrders(orders: OrderEntity[]) { - return await this.execute(this.saveBulkOrders.name, this.ordersRepository.saveBulkOrders(orders)); + async saveBulkOrdersOrUpdateAndFillId(orders: OrderEntity[]) { + return await this.execute(this.saveBulkOrdersOrUpdateAndFillId.name, this.ordersRepository.saveBulkOrdersOrUpdateAndFillId(orders)); } async updateOrderWithStatus(order: OrderEntity, status: OrderStatusEnum) { @@ -640,8 +651,11 @@ export class PersistenceService { return await this.execute(this.insertAuction.name, this.auctionsRepository.insertAuction(auction)); } - async saveBulkAuctions(auctions: AuctionEntity[]): Promise { - return await this.execute(this.saveBulkAuctions.name, this.auctionsRepository.saveBulkAuctions(auctions)); + async saveBulkAuctionsOrUpdateAndFillId(auctions: AuctionEntity[]): Promise { + return await this.execute( + this.saveBulkAuctionsOrUpdateAndFillId.name, + this.auctionsRepository.saveBulkAuctionsOrUpdateAndFillId(auctions), + ); } async rollbackAuctionAndOrdersByHash(blockHash: string): Promise { @@ -701,8 +715,8 @@ export class PersistenceService { return await this.execute(this.saveOffer.name, this.offersRepository.saveOffer(offer)); } - async saveBulkOffers(offers: OfferEntity[]): Promise { - return await this.execute(this.saveBulkOffers.name, this.offersRepository.saveBulkOffers(offers)); + async saveBulkOffersOrUpdateAndFillId(offers: OfferEntity[]): Promise { + return await this.execute(this.saveBulkOffersOrUpdateAndFillId.name, this.offersRepository.saveBulkOffersOrUpdateAndFillId(offers)); } async getOffersThatReachedDeadline(): Promise { diff --git a/src/config/default.json b/src/config/default.json index caaf03bfa..972b09535 100644 --- a/src/config/default.json +++ b/src/config/default.json @@ -76,7 +76,8 @@ "defaultPageOffset": 0, "defaultPageSize": 10, "dbMaxDenominatedValue": 99999999999999999, - "dbMaxTagLength": 20 + "dbMaxTagLength": 20, + "marketplaceReindexDataMaxInMemoryItems": 25000 }, "elasticDictionary": { "scamInfo": { diff --git a/src/modules/marketplaces/marketplaces-reindex-events-summary.service.ts b/src/modules/marketplaces/marketplaces-reindex-events-summary.service.ts index 9afba12dc..51057d11e 100644 --- a/src/modules/marketplaces/marketplaces-reindex-events-summary.service.ts +++ b/src/modules/marketplaces/marketplaces-reindex-events-summary.service.ts @@ -40,13 +40,15 @@ export class MarketplacesReindexEventsSummaryService { }); const tx = eventsSet[0].isTx ? eventsSet[0] : undefined; + const eventsStartIdx = tx ? 1 : 0; + + const txData = tx?.data?.txData; + if (eventsOrderedByOrderAsc.length === 1 && tx) { - return [undefined, undefined]; + return [undefined, txData]; } - const eventsStartIdx = tx ? 1 : 0; - - return [eventsOrderedByOrderAsc.slice(eventsStartIdx), tx?.data?.txData]; + return [eventsOrderedByOrderAsc.slice(eventsStartIdx), txData]; } private getEventSummary(event: MarketplaceEventsEntity, txData: MarketplaceTransactionData, marketplace: Marketplace): any { diff --git a/src/modules/marketplaces/marketplaces-reindex.service.ts b/src/modules/marketplaces/marketplaces-reindex.service.ts index 7ec907d54..8acd478e1 100644 --- a/src/modules/marketplaces/marketplaces-reindex.service.ts +++ b/src/modules/marketplaces/marketplaces-reindex.service.ts @@ -28,6 +28,7 @@ import { DateUtils } from 'src/utils/date-utils'; import { Locker } from '@multiversx/sdk-nestjs-common'; import { TagEntity } from 'src/db/auctions/tags.entity'; import { MarketplaceTypeEnum } from './models/MarketplaceType.enum'; +import { OrdersService } from '../orders/order.service'; import { Token } from '../usdPrice/Token.model'; @Injectable() @@ -37,6 +38,7 @@ export class MarketplacesReindexService { private readonly marketplacesService: MarketplacesService, private readonly usdPriceService: UsdPriceService, private readonly auctionSetterService: AuctionsSetterService, + private readonly ordersService: OrdersService, private readonly marketplacesReindexEventsSummaryService: MarketplacesReindexEventsSummaryService, private readonly reindexAuctionStartedHandler: ReindexAuctionStartedHandler, private readonly reindexAuctionBidHandler: ReindexAuctionBidHandler, @@ -60,16 +62,7 @@ export class MarketplacesReindexService { try { let marketplaceReindexStates: MarketplaceReindexState[] = await this.getInitialMarketplaceReindexStates(input); - await this.processMarketplaceEventsInBatches(marketplaceReindexStates, input); - - marketplaceReindexStates.map((s) => - s.setStateItemsToExpiredIfOlderThanTimestamp(input.beforeTimestamp ?? DateUtils.getCurrentTimestamp()), - ); - - for (let state of marketplaceReindexStates) { - await this.populateAuctionAssetTags(state.auctions); - await this.addMarketplaceStateToDb(state); - } + await this.processMarketplaceEventsInBatchesAndAddToDb(marketplaceReindexStates, input); this.logger.log(`Reindexing marketplace data/state for ${input.marketplaceAddress} ended`); } catch (error) { @@ -115,7 +108,7 @@ export class MarketplacesReindexService { return marketplaceReindexStates; } - private async processMarketplaceEventsInBatches( + private async processMarketplaceEventsInBatchesAndAddToDb( marketplaceReindexStates: MarketplaceReindexState[], input: MarketplaceReindexDataArgs, ): Promise { @@ -154,6 +147,8 @@ export class MarketplacesReindexService { isFinalBatch, ); + await this.addInactiveStateItemsToDb(marketplaceReindexStates, isFinalBatch); + if (processInNextBatch.length > 0) { this.logger.warn(`Could not handle ${processInNextBatch.length} events`); } @@ -216,12 +211,8 @@ export class MarketplacesReindexService { eventOrdersAndTx, ); - if (!marketplaceReindexStates[0].isFullStateInMemory) { - await Promise.all( - marketplaceReindexStates.map((state) => { - return this.getStateFromDbIfMissing(state, eventsSetSummaries); - }), - ); + for (const state of marketplaceReindexStates) { + await this.getStateFromDbIfMissing(state, eventsSetSummaries); } const areMultipleMarketplaces = marketplaceReindexStates.length !== 1; @@ -244,15 +235,19 @@ export class MarketplacesReindexService { } private async getStateFromDbIfMissing(marketplaceReindexState: MarketplaceReindexState, eventsSetSummaries: any[]): Promise { - const [missingAuctionIds, missingOfferIds, missingStateForIdentifiers] = this.getMissingStateIds( + if (marketplaceReindexState.isFullStateInMemory) { + return; + } + + const [missingMarketplaceAuctionsIds, missingMarketplaceOffersIds, missingStateForIdentifiers] = this.getMissingStateIds( marketplaceReindexState, eventsSetSummaries, ); let auctions: AuctionEntity[]; - if (missingAuctionIds.length > 0) { + if (missingMarketplaceAuctionsIds.length > 0) { auctions = await this.persistenceService.getBulkAuctionsByAuctionIdsAndMarketplace( - missingAuctionIds, + missingMarketplaceAuctionsIds, marketplaceReindexState.marketplace.key, ); } else if (missingStateForIdentifiers.length > 0) { @@ -262,14 +257,21 @@ export class MarketplacesReindexService { ); } - const auctionIds = auctions?.map((a) => a.id); + const missingAuctionsIds = auctions?.map((a) => a.id); + const [orders, offers] = await Promise.all([ - auctionIds?.length > 0 ? this.persistenceService.getOrdersByAuctionIds(auctionIds) : [], - missingOfferIds?.length > 0 - ? this.persistenceService.getBulkOffersByOfferIdsAndMarketplace(missingOfferIds, marketplaceReindexState.marketplace.key) + missingAuctionsIds?.length > 0 ? this.ordersService.getOrdersByAuctionIds(missingAuctionsIds) : [], + missingMarketplaceOffersIds?.length > 0 + ? this.persistenceService.getBulkOffersByOfferIdsAndMarketplace( + missingMarketplaceOffersIds, + marketplaceReindexState.marketplace.key, + ) : [], ]); + if (auctions?.length > 0) { + marketplaceReindexState.auctions = marketplaceReindexState.auctions.concat(auctions); + } if (orders?.length > 0) { marketplaceReindexState.orders = marketplaceReindexState.orders.concat(orders); } @@ -288,7 +290,11 @@ export class MarketplacesReindexService { continue; } - if (eventsSetSummaries[i].auctionId && eventsSetSummaries[i].action !== AssetActionEnum.StartedAuction) { + if ( + eventsSetSummaries[i].auctionId && + eventsSetSummaries[i].action !== AssetActionEnum.StartedAuction && + eventsSetSummaries[i].action !== AssetOfferEnum.Created + ) { const auctionIndex = marketplaceReindexState.marketplace.key !== ELRONDNFTSWAP_KEY ? marketplaceReindexState.getAuctionIndexByAuctionId(eventsSetSummaries[i].auctionId) @@ -346,7 +352,7 @@ export class MarketplacesReindexService { break; } case AssetActionEnum.Updated: { - const paymentToken = await this.usdPriceService.getToken(eventsSetSummary.paymentToken); + const [paymentToken] = await this.getPaymentTokenAndNonce(marketplaceReindexState, eventsSetSummary); this.reindexAuctionUpdatedHandler.handle(marketplaceReindexState, eventsSetSummary, paymentToken.decimals); break; } @@ -409,102 +415,105 @@ export class MarketplacesReindexService { } } - private async addInactiveStateItemsToDb(marketplaceReindexStates: MarketplaceReindexState[]): Promise { - for (const marketplaceReindexState of marketplaceReindexStates) { - marketplaceReindexState.setStateItemsToExpiredIfOlderThanTimestamp(DateUtils.getCurrentTimestamp()); + private async addInactiveStateItemsToDb(marketplaceReindexStates: MarketplaceReindexState[], isFinalBatch?: boolean): Promise { + try { + for (const marketplaceReindexState of marketplaceReindexStates) { + marketplaceReindexState.setStateItemsToExpiredIfOlderThanTimestamp(DateUtils.getCurrentTimestamp()); - const [inactiveAuctions, inactiveOrders, inactiveOffers] = marketplaceReindexState.popInactiveItems(); + let [inactiveAuctions, inactiveOrders, inactiveOffers] = isFinalBatch + ? marketplaceReindexState.popAllItems() + : marketplaceReindexState.popInactiveItems(); - await this.populateAuctionAssetTags(inactiveAuctions); + if (inactiveAuctions.length === 0 && inactiveOffers.length === 0) { + continue; + } - for (let i = 0; i < inactiveOrders.length; i++) { - inactiveOrders[i].auctionId = inactiveAuctions.findIndex((a) => a.id === inactiveOrders[i].auctionId); - delete inactiveOrders[i].id; - } + marketplaceReindexState.isFullStateInMemory = false; - for (let i = 0; i < inactiveAuctions.length; i++) { - delete inactiveAuctions[i].id; - } + await this.populateAuctionMissingAssetTags(inactiveAuctions); - await this.auctionSetterService.saveBulkAuctions(inactiveAuctions); - - let tags: TagEntity[] = []; - inactiveAuctions.map((auction) => { - const assetTags = auction.tags.split(','); - assetTags.map((assetTag) => { - if (assetTag !== '') { - tags.push( - new TagEntity({ - auctionId: auction.id, - tag: assetTag.trim().slice(0, constants.dbMaxTagLength), - auction: auction, - }), - ); + for (let i = 0; i < inactiveOrders.length; i++) { + if (inactiveOrders[i].auctionId < 0) { + const auctionIndex = inactiveAuctions.findIndex((a) => a.id === inactiveOrders[i].auctionId); + if (auctionIndex !== -1) { + inactiveOrders[i].auctionId = -inactiveAuctions[auctionIndex].marketplaceAuctionId; + } else { + this.logger.warn(`Corresponding auction not found`); + } } - }); - }); - const saveTagsPromise = this.persistenceService.saveTags(tags); + if (inactiveOrders[i].id < 0) { + delete inactiveOrders[i].id; + } + } - for (let i = 0; i < inactiveOrders.length; i++) { - inactiveOrders[i].auction = inactiveAuctions[inactiveOrders[i].auctionId]; - inactiveOrders[i].auctionId = inactiveAuctions[inactiveOrders[i].auctionId].id; - } - inactiveOffers.map((o) => delete o.id); + for (let i = 0; i < inactiveAuctions.length; i++) { + if (inactiveAuctions[i].id < 0) { + delete inactiveAuctions[i].id; + } + } - await Promise.all([ - saveTagsPromise, - this.persistenceService.saveBulkOrders(inactiveOrders), - this.persistenceService.saveBulkOffers(inactiveOffers), - ]); - } - } + await this.auctionSetterService.saveBulkAuctionsOrUpdateAndFillId(inactiveAuctions); + + let tags: TagEntity[] = []; + inactiveAuctions.map((auction) => { + const assetTags = auction.tags.split(','); + assetTags.map((assetTag) => { + if (assetTag !== '') { + tags.push( + new TagEntity({ + auctionId: auction.id, + tag: assetTag.trim().slice(0, constants.dbMaxTagLength), + auction: auction, + }), + ); + } + }); + }); - private async addMarketplaceStateToDb(marketplaceReindexState: MarketplaceReindexState): Promise { - marketplaceReindexState.auctions.map((a) => { - delete a.id; - }); - marketplaceReindexState.orders.map((o) => delete o.id); - marketplaceReindexState.offers.map((o) => delete o.id); + const saveTagsPromise = this.persistenceService.saveTagsOrIgnore(tags); + + for (let i = 0; i < inactiveOrders.length; i++) { + if (inactiveOrders[i].auctionId < 0) { + const auctionIndex = inactiveAuctions.findIndex((a) => a.marketplaceAuctionId === -inactiveOrders[i].auctionId); + if (auctionIndex === -1) { + this.logger.warn( + `Auction for ${marketplaceReindexState.marketplace.key} with marketplaceAuctionId ${-inactiveOrders[i] + .auctionId} was not found`, + ); + inactiveOrders.splice(i--, 1); + continue; + } + inactiveOrders[i].auction = inactiveAuctions[auctionIndex]; + inactiveOrders[i].auctionId = inactiveAuctions[auctionIndex].id; + } + } - await this.auctionSetterService.saveBulkAuctions(marketplaceReindexState.auctions); + inactiveOffers.map((o) => { + if (o.id < 0) { + delete o.id; + } + }); - for (let i = 0; i < marketplaceReindexState.orders.length; i++) { - marketplaceReindexState.orders[i].auction = marketplaceReindexState.auctions[marketplaceReindexState.orders[i].auctionId]; - marketplaceReindexState.orders[i].auctionId = marketplaceReindexState.auctions[marketplaceReindexState.orders[i].auctionId].id; + await Promise.all([ + saveTagsPromise, + this.persistenceService.saveBulkOrdersOrUpdateAndFillId(inactiveOrders), + this.persistenceService.saveBulkOffersOrUpdateAndFillId(inactiveOffers), + ]); + } + } catch (error) { + throw error; } - - let tags: TagEntity[] = []; - marketplaceReindexState.auctions.map((auction) => { - const assetTags = auction.tags.split(','); - assetTags.map((assetTag) => { - if (assetTag !== '') { - tags.push( - new TagEntity({ - auctionId: auction.id, - tag: assetTag.trim().slice(0, constants.dbMaxTagLength), - auction: auction, - }), - ); - } - }); - }); - - await Promise.all([ - this.persistenceService.saveBulkOrders(marketplaceReindexState.orders), - this.persistenceService.saveBulkOffers(marketplaceReindexState.offers), - this.persistenceService.saveTags(tags), - ]); } - private async populateAuctionAssetTags(auctions: AuctionEntity[]): Promise { + private async populateAuctionMissingAssetTags(auctions: AuctionEntity[]): Promise { const batchSize = constants.getNftsFromApiBatchSize; for (let i = 0; i < auctions.length; i += batchSize) { const assetsWithNoTagsIdentifiers = [ ...new Set( auctions .slice(i, i + batchSize) - .filter((a) => a.tags === '') + .filter((a) => a.tags === '' && a.id < 0) .map((a) => a.identifier), ), ]; diff --git a/src/modules/marketplaces/marketplaces.module.ts b/src/modules/marketplaces/marketplaces.module.ts index 3451f7944..2c373162c 100644 --- a/src/modules/marketplaces/marketplaces.module.ts +++ b/src/modules/marketplaces/marketplaces.module.ts @@ -26,6 +26,7 @@ import { ReindexAuctionPriceUpdatedHandler } from './marketplaces-reindex-handle import { ReindexGlobalOfferAcceptedHandler } from './marketplaces-reindex-handlers/reindex-global-offer-accepted.handler'; import { ReindexAuctionUpdatedHandler } from './marketplaces-reindex-handlers/reindex-auction-updated.handler'; import { MarketplacesMutationsResolver } from './marketplaces-mutations.resolver'; +import { OrdersModuleGraph } from '../orders/orders.module'; @Module({ providers: [ @@ -58,6 +59,7 @@ import { MarketplacesMutationsResolver } from './marketplaces-mutations.resolver MxCommunicationModule, forwardRef(() => CommonModule), forwardRef(() => AuctionsModuleGraph), + forwardRef(() => OrdersModuleGraph), forwardRef(() => OffersModuleGraph), ], exports: [MarketplacesService, MarketplaceEventsIndexingService, MarketplacesReindexService], diff --git a/src/modules/marketplaces/models/MarketplaceReindexState.ts b/src/modules/marketplaces/models/MarketplaceReindexState.ts index 9205231a5..dbcffd7d4 100644 --- a/src/modules/marketplaces/models/MarketplaceReindexState.ts +++ b/src/modules/marketplaces/models/MarketplaceReindexState.ts @@ -1,5 +1,6 @@ import { ObjectType } from '@nestjs/graphql'; import BigNumber from 'bignumber.js'; +import { constants } from 'src/config'; import { AuctionEntity } from 'src/db/auctions'; import { OfferEntity } from 'src/db/offers'; import { OrderEntity } from 'src/db/orders'; @@ -20,10 +21,26 @@ export class MarketplaceReindexState { orders: OrderEntity[] = []; offers: OfferEntity[] = []; + private auctionsTemporaryIdCounter = -1; + private ordersTemporaryIdCounter = -1; + private offersTemporaryIdCounter = -1; + constructor(init?: Partial) { Object.assign(this, init); } + getNewAuctionId(): number { + return this.auctionsTemporaryIdCounter--; + } + + getNewOrderId(): number { + return this.ordersTemporaryIdCounter--; + } + + getNewOfferId(): number { + return this.offersTemporaryIdCounter--; + } + isCollectionListed(collection: string): boolean { return this.listedCollections.includes(collection); } @@ -44,14 +61,14 @@ export class MarketplaceReindexState { const modifiedDate = DateUtils.getUtcDateFromTimestamp(input.timestamp); const price = input.price ?? input.currentBid; return new OrderEntity({ - id: this.orders.length, + id: this.getNewOrderId(), creationDate: modifiedDate, modifiedDate, auctionId: this.auctions[auctionIndex].id, ownerAddress: input.address, priceToken: paymentToken.identifier, priceNonce: paymentNonce ?? 0, - priceAmount: price !== '0' ? price : this.auctions[auctionIndex].maxBid, + priceAmount: new BigNumber(price !== '0' ? price : this.auctions[auctionIndex].maxBid).toFixed(), priceAmountDenominated: price !== '0' ? BigNumberUtils.denominateAmount(price, paymentToken.decimals) : this.auctions[auctionIndex].maxBidDenominated, blockHash: input.blockHash ?? '', @@ -94,6 +111,16 @@ export class MarketplaceReindexState { this.setOffersToExpiredIfOlderThanTimestamp(timestamp); } + popAllItems(): [AuctionEntity[], OrderEntity[], OfferEntity[]] { + const inactiveAuctions = this.auctions; + const inactiveOrders = this.orders; + const inactiveOffers = this.offers; + this.auctions = []; + this.orders = []; + this.offers = []; + return [inactiveAuctions, inactiveOrders, inactiveOffers]; + } + popInactiveItems(): [AuctionEntity[], OrderEntity[], OfferEntity[]] { const inactiveAuctionStatuses = [AuctionStatusEnum.Closed, AuctionStatusEnum.Ended]; const inactiveOfferStatuses = [OfferStatusEnum.Accepted, OfferStatusEnum.Closed, OfferStatusEnum.Expired]; @@ -102,33 +129,86 @@ export class MarketplaceReindexState { let inactiveOrders = []; let inactiveOffers = []; + // if (this.marketplace.key === 'elrondapes') + // console.log( + // `state`, + // JSON.stringify(this.auctions), + // JSON.stringify(this.orders), + // JSON.stringify(this.offers), + // ); + // if (this.marketplace.key === 'elrondapes') + // console.log( + // `inactive`, + // JSON.stringify(inactiveAuctions), + // JSON.stringify(inactiveOrders), + // JSON.stringify(inactiveOffers), + // ); + + if (this.marketplace.key === 'elrondapes') { + console.log( + `orders state`, + this.orders.map((o) => o.id), + this.orders.map((o) => o.auctionId), + ); + } + for (let i = 0; i < this.auctions.length; i++) { - if (inactiveAuctionStatuses.includes(this.auctions[i].status)) { - inactiveAuctions.push(this.auctions[i]); - delete this.auctions[i]; + const isInactiveAuction = inactiveAuctionStatuses.includes(this.auctions[i].status); + const isOldAuction = + this.auctions.length > constants.marketplaceReindexDataMaxInMemoryItems && + i < this.auctions.length - constants.marketplaceReindexDataMaxInMemoryItems; + + if (isInactiveAuction || isOldAuction) { + inactiveAuctions.push(this.auctions.splice(i--, 1)[0]); } } - this.auctions = this.auctions.filter((a) => a); for (let i = 0; i < this.orders.length; i++) { - if (inactiveAuctions.findIndex((a) => a.id === this.orders[i].auctionId) !== -1) { - inactiveOrders.push(this.orders[i]); - delete this.orders[i]; + const isInactiveOrOldOrder = inactiveAuctions.findIndex((a) => a.id === this.orders[i].auctionId) !== -1; + + if (isInactiveOrOldOrder) { + inactiveOrders.push(this.orders.splice(i--, 1)[0]); } } - this.orders = this.orders.filter((o) => o); for (let i = 0; i < this.offers.length; i++) { - if (inactiveOfferStatuses.includes(this.offers[i].status)) { - inactiveOffers.push(this.offers[i]); - delete this.offers[i]; + const isInactiveOffer = inactiveOfferStatuses.includes(this.offers[i].status); + const isOldOffer = + this.offers.length > constants.marketplaceReindexDataMaxInMemoryItems && + i < this.offers.length - constants.marketplaceReindexDataMaxInMemoryItems; + + if (isInactiveOffer || isOldOffer) { + inactiveOffers.push(this.offers.splice(i--, 1)[0]); } } - this.offers = this.offers.filter((o) => o); return [inactiveAuctions, inactiveOrders, inactiveOffers]; } + deleteAuctionIfDuplicates(marketplaceAuctionId: number) { + if (this.isFullStateInMemory) { + return; + } + + let index; + do { + index = this.auctions.findIndex((a) => a.marketplaceAuctionId === marketplaceAuctionId); + this.auctions.splice(index, 1); + } while (index !== -1); + } + + deleteOfferIfDuplicates(marketplaceOfferId: number) { + if (this.isFullStateInMemory) { + return; + } + + let index; + do { + index = this.offers.findIndex((a) => a.marketplaceOfferId === marketplaceOfferId); + this.offers.splice(index, 1); + } while (index !== -1); + } + private setAuctionsAndOrdersToExpiredIfOlderThanTimestamp(timestamp: number): void { const runningAuctions = this.auctions?.filter((a) => a.status === AuctionStatusEnum.Running); for (let i = 0; i < runningAuctions.length; i++) { From 27948d4a384bd296a983892d8baf52bdfb7e7afa Mon Sep 17 00:00:00 2001 From: danielailie Date: Thu, 9 Nov 2023 15:49:29 +0200 Subject: [PATCH 04/18] Update reindex data marketplace --- src/config/default.json | 2 +- .../admins/admin-operations.resolver.ts | 2 +- ...etplaces-reindex-events-summary.service.ts | 23 ++++++++----------- .../marketplaces-reindex.service.ts | 23 +++++++++++-------- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/src/config/default.json b/src/config/default.json index 972b09535..d4f606c70 100644 --- a/src/config/default.json +++ b/src/config/default.json @@ -69,7 +69,7 @@ "updateAllNftTraitsBatchSize": 10000, "getNftsForScamInfoBatchSize": 100, "elasticMaxBatch": 10000, - "dbBatch": 1000, + "dbBatch": 10000, "complexityLevel": 200, "getLogsFromElasticBatchSize": 1000, "dbMaxTimestamp": 2147483647, diff --git a/src/modules/admins/admin-operations.resolver.ts b/src/modules/admins/admin-operations.resolver.ts index 1955fdb35..273bf6bf7 100644 --- a/src/modules/admins/admin-operations.resolver.ts +++ b/src/modules/admins/admin-operations.resolver.ts @@ -147,7 +147,7 @@ export class AdminOperationsResolver { } @Mutation(() => Boolean) - @UseGuards(JwtOrNativeAuthGuard, GqlAdminAuthGuard) + // @UseGuards(JwtOrNativeAuthGuard, GqlAdminAuthGuard) async reindexMarketplaceData( @Args('input') input: MarketplaceReindexDataArgs, diff --git a/src/modules/marketplaces/marketplaces-reindex-events-summary.service.ts b/src/modules/marketplaces/marketplaces-reindex-events-summary.service.ts index 51057d11e..f6dc28eff 100644 --- a/src/modules/marketplaces/marketplaces-reindex-events-summary.service.ts +++ b/src/modules/marketplaces/marketplaces-reindex-events-summary.service.ts @@ -34,21 +34,18 @@ export class MarketplacesReindexEventsSummaryService { }); } - private getEventsAndTxData(eventsSet: MarketplaceEventsEntity[]): [MarketplaceEventsEntity[], MarketplaceTransactionData] { - const eventsOrderedByOrderAsc = eventsSet.sort((a, b) => { - return a.eventOrder - b.eventOrder; - }); - const tx = eventsSet[0].isTx ? eventsSet[0] : undefined; - - const eventsStartIdx = tx ? 1 : 0; + private getEventsAndTxData( + eventsSet: MarketplaceEventsEntity[], + ): [MarketplaceEventsEntity[] | undefined, MarketplaceTransactionData | undefined] { + const tx = eventsSet.find((event) => event.isTx); - const txData = tx?.data?.txData; - - if (eventsOrderedByOrderAsc.length === 1 && tx) { - return [undefined, txData]; + if (eventsSet.length === 1 && tx) { + return [undefined, tx.data?.txData]; } - return [eventsOrderedByOrderAsc.slice(eventsStartIdx), txData]; + const eventsWithoutTx = eventsSet.filter((event) => !event.isTx); + + return [eventsWithoutTx.length > 0 ? eventsWithoutTx : undefined, tx?.data?.txData]; } private getEventSummary(event: MarketplaceEventsEntity, txData: MarketplaceTransactionData, marketplace: Marketplace): any { @@ -116,7 +113,7 @@ export class MarketplacesReindexEventsSummaryService { } } default: { - throw new Error(`Unhandled marketplace event - ${event.data.eventData.identifier}`); + throw new Error(`Unhandled marketplace event - ${event?.data?.eventData?.identifier}`); } } } diff --git a/src/modules/marketplaces/marketplaces-reindex.service.ts b/src/modules/marketplaces/marketplaces-reindex.service.ts index 8acd478e1..201bfe28e 100644 --- a/src/modules/marketplaces/marketplaces-reindex.service.ts +++ b/src/modules/marketplaces/marketplaces-reindex.service.ts @@ -56,6 +56,7 @@ export class MarketplacesReindexService { ) {} async reindexMarketplaceData(input: MarketplaceReindexDataArgs): Promise { + this.logger.log(`Reindexing marketplace data/state for ${input.marketplaceAddress} started`); await Locker.lock( `Reindex marketplace data/state for ${input.marketplaceAddress}`, async () => { @@ -120,6 +121,7 @@ export class MarketplacesReindexService { do { let batch = await nextBatchPromise; + console.log({ batch: batch.length, afterTimestamp }); if (!batch || batch.length === 0) { break; } @@ -132,11 +134,6 @@ export class MarketplacesReindexService { processInNextBatch.concat(batch), ); - processInNextBatch = await this.processEventsBatchAndReturnUnprocessedEvents( - marketplaceReindexStates, - processInNextBatch.concat(batch), - ); - await this.addInactiveStateItemsToDb(marketplaceReindexStates); } while (input.beforeTimestamp ? afterTimestamp < input.beforeTimestamp : true); @@ -181,14 +178,20 @@ export class MarketplacesReindexService { batch: MarketplaceEventsEntity[], isFinalBatch?: boolean, ): Promise { - let unprocessedEvents: MarketplaceEventsEntity[] = [...batch]; + const eventsMap: Map = new Map(); - while (unprocessedEvents.length > 0) { - const txHash = unprocessedEvents[0].txHash; + batch.forEach((event) => { + const txHash = event.txHash || event.originalTxHash || ''; // Adjust accordingly + if (!eventsMap.has(txHash)) { + eventsMap.set(txHash, []); + } + eventsMap.get(txHash).push(event); + }); - const eventOrdersAndTx = unprocessedEvents.filter((event) => event.txHash === txHash || event.originalTxHash === txHash); + let unprocessedEvents: MarketplaceEventsEntity[] = [...batch]; - const isAnotherEventsSetInBatch = unprocessedEvents.find((e) => e.timestamp > unprocessedEvents[0].timestamp); + for (const [txHash, eventOrdersAndTx] of eventsMap) { + const isAnotherEventsSetInBatch = unprocessedEvents.find((e) => e.timestamp > eventOrdersAndTx[0].timestamp); if (!isFinalBatch && (eventOrdersAndTx.length === unprocessedEvents.length || !isAnotherEventsSetInBatch)) { return unprocessedEvents; From 368b1f3ad5933d4dd8a279daa8e060f415884ef1 Mon Sep 17 00:00:00 2001 From: danielailie Date: Fri, 10 Nov 2023 12:27:36 +0200 Subject: [PATCH 05/18] Add extra logs --- .../marketplaces-reindex.service.ts | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/modules/marketplaces/marketplaces-reindex.service.ts b/src/modules/marketplaces/marketplaces-reindex.service.ts index 201bfe28e..95c93f657 100644 --- a/src/modules/marketplaces/marketplaces-reindex.service.ts +++ b/src/modules/marketplaces/marketplaces-reindex.service.ts @@ -133,8 +133,12 @@ export class MarketplacesReindexService { marketplaceReindexStates, processInNextBatch.concat(batch), ); - - await this.addInactiveStateItemsToDb(marketplaceReindexStates); + console.log({ + auctions: marketplaceReindexStates[0].auctions.length, + orders: marketplaceReindexStates[0].orders.length, + offers: marketplaceReindexStates[0].offers.length, + }); + // await this.addInactiveStateItemsToDb(marketplaceReindexStates); } while (input.beforeTimestamp ? afterTimestamp < input.beforeTimestamp : true); const isFinalBatch = true; @@ -402,6 +406,15 @@ export class MarketplacesReindexService { 0, ]; } + if (!paymentTokenIdentifier) { + return [ + new Token({ + identifier: mxConfig.egld, + decimals: mxConfig.decimals, + }), + paymentNonce, + ]; + } const paymentToken = await this.usdPriceService.getToken(paymentTokenIdentifier); if (!paymentToken) { return [ @@ -420,6 +433,7 @@ export class MarketplacesReindexService { private async addInactiveStateItemsToDb(marketplaceReindexStates: MarketplaceReindexState[], isFinalBatch?: boolean): Promise { try { + console.log('save to db'); for (const marketplaceReindexState of marketplaceReindexStates) { marketplaceReindexState.setStateItemsToExpiredIfOlderThanTimestamp(DateUtils.getCurrentTimestamp()); @@ -427,6 +441,12 @@ export class MarketplacesReindexService { ? marketplaceReindexState.popAllItems() : marketplaceReindexState.popInactiveItems(); + console.log({ + inactiveOrders: inactiveOrders.length, + inactiveAuctions: inactiveAuctions.length, + inactiveOffers: inactiveOffers.length, + }); + if (inactiveAuctions.length === 0 && inactiveOffers.length === 0) { continue; } @@ -497,6 +517,7 @@ export class MarketplacesReindexService { delete o.id; } }); + console.log({ orders: inactiveOrders.length, auctions: inactiveAuctions.length, offers: inactiveOffers.length }); await Promise.all([ saveTagsPromise, From 4a79d29c053456914505e1a140705c865c8cb6c2 Mon Sep 17 00:00:00 2001 From: danielailie Date: Mon, 13 Nov 2023 14:05:29 +0200 Subject: [PATCH 06/18] Update indexing model --- src/db/auctions/auctions.repository.ts | 114 ++++++++---------- .../reindex-auction-bid.handler.ts | 10 +- .../reindex-auction-bought.handler.ts | 12 +- .../reindex-auction-closed.handler.ts | 5 +- .../reindex-auction-ended.handler.ts | 4 +- .../marketplaces-reindex.service.ts | 34 +++--- .../models/MarketplaceReindexState.ts | 62 +++++++--- 7 files changed, 125 insertions(+), 116 deletions(-) diff --git a/src/db/auctions/auctions.repository.ts b/src/db/auctions/auctions.repository.ts index 0ac32043e..768b3dfe5 100644 --- a/src/db/auctions/auctions.repository.ts +++ b/src/db/auctions/auctions.repository.ts @@ -421,10 +421,7 @@ export class AuctionsRepository { return await this.auctionsRepository.save(auction); } - async getBulkAuctionsByMarketplaceAndAuctionIds( - marketplaceKey: string, - marketplaceAuctionIds: number[], - ): Promise { + async getBulkAuctionsByMarketplaceAndAuctionIds(marketplaceKey: string, marketplaceAuctionIds: number[]): Promise { return await this.auctionsRepository .createQueryBuilder('a') .select('id, marketplaceAuctionId') @@ -436,68 +433,59 @@ export class AuctionsRepository { .execute(); } - async saveBulkAuctionsOrUpdateAndFillId( - auctions: AuctionEntity[], - ): Promise { + async saveBulkAuctionsOrUpdateAndFillId(auctions: AuctionEntity[]): Promise { if (auctions.length === 0) { return; } - const saveOrUpdateResponse = await this.auctionsRepository - .createQueryBuilder() - .insert() - .into('auctions') - .values(auctions) - .orUpdate({ - overwrite: [ - 'creationDate', - 'modifiedDate', - 'collection', - 'nrAuctionedTokens', - 'identifier', - 'nonce', - 'status', - 'type', - 'paymentToken', - 'paymentNonce', - 'ownerAddress', - 'minBidDiff', - 'minBid', - 'minBidDenominated', - 'maxBid', - 'maxBidDenominated', - 'startDate', - 'endDate', - 'tags', - 'blockHash', - ], - conflict_target: ['marketplaceAuctionId', 'marketplaceKey'], - }) - .updateEntity(false) - .execute(); - if ( - saveOrUpdateResponse.identifiers.length === 0 || - auctions.findIndex((a) => a.id === undefined) !== -1 - ) { - const dbAuctions = await this.getBulkAuctionsByMarketplaceAndAuctionIds( - auctions?.[0]?.marketplaceKey, - auctions?.map((a) => a.marketplaceAuctionId), - ); - for (let i = 0; i < dbAuctions.length; i++) { - const auctionIndex = auctions.findIndex( - (a) => a.marketplaceAuctionId === dbAuctions[i].marketplaceAuctionId, - ); - auctions[auctionIndex].id = dbAuctions[i].id; - } - } - if (auctions.findIndex((a) => a.id === undefined) !== -1) { - const wtf = auctions - .filter((a) => a.id === undefined) - .map((a) => a.marketplaceAuctionId); - const duplicate = auctions.filter( - (a) => a.marketplaceAuctionId === 31434, - ); - throw new Error(`oooppps ${JSON.stringify(duplicate)}`); - } + const response = await this.auctionsRepository.upsert(auctions, { conflictPaths: ['marketplaceAuctionId', 'marketplaceKey'] }); + + // const saveOrUpdateResponse = await this.auctionsRepository + // .createQueryBuilder() + // .insert() + // .into('auctions') + // .values(auctions) + // .orUpdate({ + // overwrite: [ + // 'creationDate', + // 'modifiedDate', + // 'collection', + // 'nrAuctionedTokens', + // 'identifier', + // 'nonce', + // 'status', + // 'type', + // 'paymentToken', + // 'paymentNonce', + // 'ownerAddress', + // 'minBidDiff', + // 'minBid', + // 'minBidDenominated', + // 'maxBid', + // 'maxBidDenominated', + // 'startDate', + // 'endDate', + // 'tags', + // 'blockHash', + // ], + // conflict_target: ['marketplaceAuctionId', 'marketplaceKey'], + // }) + // .updateEntity(false) + // .execute(); + // if (saveOrUpdateResponse.identifiers.length === 0 || auctions.findIndex((a) => a.id === undefined) !== -1) { + // const dbAuctions = await this.getBulkAuctionsByMarketplaceAndAuctionIds( + // auctions?.[0]?.marketplaceKey, + // auctions?.map((a) => a.marketplaceAuctionId), + // ); + // for (let i = 0; i < dbAuctions.length; i++) { + // const auctionIndex = auctions.findIndex((a) => a.marketplaceAuctionId === dbAuctions[i].marketplaceAuctionId); + // auctions[auctionIndex].id = dbAuctions[i].id; + // } + // } + // if (auctions.findIndex((a) => a.id === undefined) !== -1) { + // const wtf = auctions.filter((a) => a.id === undefined).map((a) => a.marketplaceAuctionId); + // const duplicate = auctions.filter((a) => a.marketplaceAuctionId === 31434); + // throw new Error(`oooppps ${JSON.stringify(duplicate)}`); + // } } async rollbackAuctionAndOrdersByHash(blockHash: string): Promise { diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bid.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bid.handler.ts index 0cf36c4de..43b5f0a3d 100644 --- a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bid.handler.ts +++ b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bid.handler.ts @@ -2,7 +2,6 @@ import { Injectable } from '@nestjs/common'; import { AuctionStatusEnum } from 'src/modules/auctions/models'; import { OrderStatusEnum } from 'src/modules/orders/models'; import { Token } from 'src/modules/usdPrice/Token.model'; -import { DateUtils } from 'src/utils/date-utils'; import { MarketplaceReindexState } from '../models/MarketplaceReindexState'; import { AuctionBidSummary } from '../models/marketplaces-reindex-events-summaries/AuctionBidSummary'; @@ -17,18 +16,15 @@ export class ReindexAuctionBidHandler { return; } - const modifiedDate = DateUtils.getUtcDateFromTimestamp(input.timestamp); - - marketplaceReindexState.setInactiveOrdersForAuction(marketplaceReindexState.auctions[auctionIndex].id, modifiedDate); - let order = marketplaceReindexState.createOrder(auctionIndex, input, OrderStatusEnum.Active, paymentToken, paymentNonce); if (order.priceAmount === marketplaceReindexState.auctions[auctionIndex].maxBid) { order.status = OrderStatusEnum.Bought; - marketplaceReindexState.auctions[auctionIndex].status = AuctionStatusEnum.Ended; - marketplaceReindexState.auctions[auctionIndex].modifiedDate = modifiedDate; + marketplaceReindexState.updateAuctionStatus(auctionIndex, input.blockHash, AuctionStatusEnum.Ended, input.timestamp); } + marketplaceReindexState.updateOrderListForAuction(auctionIndex, order); + marketplaceReindexState.orders.push(order); } } diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bought.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bought.handler.ts index e27ea65b6..d4dba7829 100644 --- a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bought.handler.ts +++ b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bought.handler.ts @@ -4,7 +4,6 @@ import { AuctionStatusEnum } from 'src/modules/auctions/models'; import { OrderStatusEnum } from 'src/modules/orders/models'; import { Token } from 'src/modules/usdPrice/Token.model'; import { ELRONDNFTSWAP_KEY } from 'src/utils/constants'; -import { DateUtils } from 'src/utils/date-utils'; import { MarketplaceReindexState } from '../models/MarketplaceReindexState'; import { AuctionBuySummary } from '../models/marketplaces-reindex-events-summaries/AuctionBuySummary'; @@ -22,10 +21,6 @@ export class ReindexAuctionBoughtHandler { return; } - const modifiedDate = DateUtils.getUtcDateFromTimestamp(input.timestamp); - - marketplaceReindexState.setInactiveOrdersForAuction(marketplaceReindexState.auctions[auctionIndex].id, modifiedDate); - const order = marketplaceReindexState.createOrder(auctionIndex, input, OrderStatusEnum.Bought, paymentToken, paymentNonce); marketplaceReindexState.orders.push(order); @@ -35,11 +30,10 @@ export class ReindexAuctionBoughtHandler { ); if (marketplaceReindexState.auctions[auctionIndex].nrAuctionedTokens === totalBought) { - marketplaceReindexState.auctions[auctionIndex].status = AuctionStatusEnum.Ended; - marketplaceReindexState.auctions[auctionIndex].modifiedDate = modifiedDate; - marketplaceReindexState.auctions[auctionIndex].blockHash = - marketplaceReindexState.auctions[auctionIndex].blockHash ?? input.blockHash; + marketplaceReindexState.updateAuctionStatus(auctionIndex, input.blockHash, AuctionStatusEnum.Ended, input.timestamp); } + + marketplaceReindexState.updateOrderListForAuction(auctionIndex, order); } private getTotalBoughtTokensForAuction(auctionId: number, orders: OrderEntity[]): number { diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-closed.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-closed.handler.ts index 9ed9bb1ad..826240c63 100644 --- a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-closed.handler.ts +++ b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-closed.handler.ts @@ -20,10 +20,9 @@ export class ReindexAuctionClosedHandler { return; } - marketplaceReindexState.auctions[auctionIndex].status = AuctionStatusEnum.Closed; - marketplaceReindexState.auctions[auctionIndex].blockHash = marketplaceReindexState.auctions[auctionIndex].blockHash ?? input.blockHash; - marketplaceReindexState.auctions[auctionIndex].modifiedDate = modifiedDate; + marketplaceReindexState.updateAuctionStatus(auctionIndex, input.blockHash, AuctionStatusEnum.Closed, input.timestamp); marketplaceReindexState.setInactiveOrdersForAuction(marketplaceReindexState.auctions[auctionIndex].id, modifiedDate); + marketplaceReindexState.setInactiveOrdersForAuctionNew(auctionIndex, modifiedDate); } } diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-ended.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-ended.handler.ts index 6230b8be3..d38ffb4a2 100644 --- a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-ended.handler.ts +++ b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-ended.handler.ts @@ -18,9 +18,7 @@ export class ReindexAuctionEndedHandler { return; } - marketplaceReindexState.auctions[auctionIndex].status = AuctionStatusEnum.Ended; - marketplaceReindexState.auctions[auctionIndex].blockHash = marketplaceReindexState.auctions[auctionIndex].blockHash ?? input.blockHash; - marketplaceReindexState.auctions[auctionIndex].modifiedDate = modifiedDate; + marketplaceReindexState.updateAuctionStatus(auctionIndex, input.blockHash, AuctionStatusEnum.Ended, input.timestamp); const winnerOrderId = marketplaceReindexState.setAuctionOrderWinnerStatusAndReturnId( marketplaceReindexState.auctions[auctionIndex].id, diff --git a/src/modules/marketplaces/marketplaces-reindex.service.ts b/src/modules/marketplaces/marketplaces-reindex.service.ts index 95c93f657..4ee0282e9 100644 --- a/src/modules/marketplaces/marketplaces-reindex.service.ts +++ b/src/modules/marketplaces/marketplaces-reindex.service.ts @@ -236,6 +236,7 @@ export class MarketplacesReindexService { await this.processEvent(marketplaceReindexStates[stateIndex], eventsSetSummaries[i]); } } catch (error) { + console.log({ error }); this.logger.warn(`Error reprocessing marketplace event ${JSON.stringify(eventsSetSummaries[i])} - ${JSON.stringify(error)}`); } } @@ -435,7 +436,7 @@ export class MarketplacesReindexService { try { console.log('save to db'); for (const marketplaceReindexState of marketplaceReindexStates) { - marketplaceReindexState.setStateItemsToExpiredIfOlderThanTimestamp(DateUtils.getCurrentTimestamp()); + // marketplaceReindexState.setStateItemsToExpiredIfOlderThanTimestamp(DateUtils.getCurrentTimestamp()); let [inactiveAuctions, inactiveOrders, inactiveOffers] = isFinalBatch ? marketplaceReindexState.popAllItems() @@ -444,6 +445,7 @@ export class MarketplacesReindexService { console.log({ inactiveOrders: inactiveOrders.length, inactiveAuctions: inactiveAuctions.length, + inactiveordersONAc: inactiveAuctions.sum((au) => au.orders?.length ?? 0), inactiveOffers: inactiveOffers.length, }); @@ -496,21 +498,21 @@ export class MarketplacesReindexService { const saveTagsPromise = this.persistenceService.saveTagsOrIgnore(tags); - for (let i = 0; i < inactiveOrders.length; i++) { - if (inactiveOrders[i].auctionId < 0) { - const auctionIndex = inactiveAuctions.findIndex((a) => a.marketplaceAuctionId === -inactiveOrders[i].auctionId); - if (auctionIndex === -1) { - this.logger.warn( - `Auction for ${marketplaceReindexState.marketplace.key} with marketplaceAuctionId ${-inactiveOrders[i] - .auctionId} was not found`, - ); - inactiveOrders.splice(i--, 1); - continue; - } - inactiveOrders[i].auction = inactiveAuctions[auctionIndex]; - inactiveOrders[i].auctionId = inactiveAuctions[auctionIndex].id; - } - } + // for (let i = 0; i < inactiveOrders.length; i++) { + // if (inactiveOrders[i].auctionId < 0) { + // const auctionIndex = inactiveAuctions.findIndex((a) => a.marketplaceAuctionId === -inactiveOrders[i].auctionId); + // if (auctionIndex === -1) { + // this.logger.warn( + // `Auction for ${marketplaceReindexState.marketplace.key} with marketplaceAuctionId ${-inactiveOrders[i] + // .auctionId} was not found`, + // ); + // inactiveOrders.splice(i--, 1); + // continue; + // } + // inactiveOrders[i].auction = inactiveAuctions[auctionIndex]; + // inactiveOrders[i].auctionId = inactiveAuctions[auctionIndex].id; + // } + // } inactiveOffers.map((o) => { if (o.id < 0) { diff --git a/src/modules/marketplaces/models/MarketplaceReindexState.ts b/src/modules/marketplaces/models/MarketplaceReindexState.ts index dbcffd7d4..390bafeeb 100644 --- a/src/modules/marketplaces/models/MarketplaceReindexState.ts +++ b/src/modules/marketplaces/models/MarketplaceReindexState.ts @@ -49,6 +49,10 @@ export class MarketplaceReindexState { return this.auctions.findIndex((a) => a.marketplaceAuctionId === auctionId); } + getAuctionByAuctionId(auctionId: number): number { + return this.auctions.findIndex((a) => a.marketplaceAuctionId === auctionId); + } + getAuctionIndexByIdentifier(identifier: string): number { return this.auctions.findIndex((a) => a.identifier === identifier && a.status === AuctionStatusEnum.Running); } @@ -97,6 +101,25 @@ export class MarketplaceReindexState { return -1; } + setAuctionOrderWinnerStatusAndeturnId(auctionId: number, status: OrderStatusEnum, modifiedDate?: Date): number { + const bids = this.orders + .filter((o) => o.auctionId === auctionId && o.status === OrderStatusEnum.Active) + .map((o) => new BigNumber(o.priceAmount)); + + if (bids.length) { + const maxBid = BigNumber.max(...bids); + const winnerOrderIndex = this.orders.findIndex( + (o) => o.auctionId === auctionId && o.status === OrderStatusEnum.Active && o.priceAmount === maxBid.toString(), + ); + this.orders[winnerOrderIndex].status = status; + if (modifiedDate) { + this.orders[winnerOrderIndex].modifiedDate = modifiedDate; + } + return this.orders[winnerOrderIndex].id; + } + return -1; + } + setInactiveOrdersForAuction(auctionId: number, modifiedDate: Date, exceptWinnerId?: number): void { this.orders ?.filter((o) => o.auctionId === auctionId && o.status === OrderStatusEnum.Active && o.id !== exceptWinnerId) @@ -106,6 +129,15 @@ export class MarketplaceReindexState { }); } + setInactiveOrdersForAuctionNew(auctionId: number, modifiedDate: Date, exceptWinnerId?: number): void { + this.auctions[auctionId]?.orders + ?.filter((o) => o.auctionId === auctionId && o.status === OrderStatusEnum.Active && o.id !== exceptWinnerId) + ?.map((o) => { + o.status = OrderStatusEnum.Inactive; + o.modifiedDate = modifiedDate; + }); + } + setStateItemsToExpiredIfOlderThanTimestamp(timestamp: number): void { this.setAuctionsAndOrdersToExpiredIfOlderThanTimestamp(timestamp); this.setOffersToExpiredIfOlderThanTimestamp(timestamp); @@ -129,21 +161,6 @@ export class MarketplaceReindexState { let inactiveOrders = []; let inactiveOffers = []; - // if (this.marketplace.key === 'elrondapes') - // console.log( - // `state`, - // JSON.stringify(this.auctions), - // JSON.stringify(this.orders), - // JSON.stringify(this.offers), - // ); - // if (this.marketplace.key === 'elrondapes') - // console.log( - // `inactive`, - // JSON.stringify(inactiveAuctions), - // JSON.stringify(inactiveOrders), - // JSON.stringify(inactiveOffers), - // ); - if (this.marketplace.key === 'elrondapes') { console.log( `orders state`, @@ -185,6 +202,21 @@ export class MarketplaceReindexState { return [inactiveAuctions, inactiveOrders, inactiveOffers]; } + public updateOrderListForAuction(auctionIndex: number, order: OrderEntity) { + this.auctions[auctionIndex].orders = this.auctions[auctionIndex].orders + ? [...this.auctions[auctionIndex].orders?.map((order) => ({ ...order, status: OrderStatusEnum.Inactive })), order] + : [order]; + } + + public updateAuctionStatus(auctionIndex: number, blockHash: string, status: AuctionStatusEnum, timestamp: number) { + this.auctions[auctionIndex] = { + ...this.auctions[auctionIndex], + status: status, + blockHash: this.auctions[auctionIndex].blockHash ?? blockHash, + modifiedDate: DateUtils.getUtcDateFromTimestamp(timestamp), + }; + } + deleteAuctionIfDuplicates(marketplaceAuctionId: number) { if (this.isFullStateInMemory) { return; From a8c0450d37c8f72112a44430c4a2b7c6b473457a Mon Sep 17 00:00:00 2001 From: danielailie Date: Mon, 13 Nov 2023 14:18:41 +0200 Subject: [PATCH 07/18] Remove save orders --- src/modules/marketplaces/marketplaces-reindex.service.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/modules/marketplaces/marketplaces-reindex.service.ts b/src/modules/marketplaces/marketplaces-reindex.service.ts index 4ee0282e9..ce9d76634 100644 --- a/src/modules/marketplaces/marketplaces-reindex.service.ts +++ b/src/modules/marketplaces/marketplaces-reindex.service.ts @@ -455,7 +455,7 @@ export class MarketplacesReindexService { marketplaceReindexState.isFullStateInMemory = false; - await this.populateAuctionMissingAssetTags(inactiveAuctions); + // await this.populateAuctionMissingAssetTags(inactiveAuctions); for (let i = 0; i < inactiveOrders.length; i++) { if (inactiveOrders[i].auctionId < 0) { @@ -496,7 +496,7 @@ export class MarketplacesReindexService { }); }); - const saveTagsPromise = this.persistenceService.saveTagsOrIgnore(tags); + // const saveTagsPromise = this.persistenceService.saveTagsOrIgnore(tags); // for (let i = 0; i < inactiveOrders.length; i++) { // if (inactiveOrders[i].auctionId < 0) { @@ -522,8 +522,8 @@ export class MarketplacesReindexService { console.log({ orders: inactiveOrders.length, auctions: inactiveAuctions.length, offers: inactiveOffers.length }); await Promise.all([ - saveTagsPromise, - this.persistenceService.saveBulkOrdersOrUpdateAndFillId(inactiveOrders), + // saveTagsPromise, + // this.persistenceService.saveBulkOrdersOrUpdateAndFillId(inactiveOrders), this.persistenceService.saveBulkOffersOrUpdateAndFillId(inactiveOffers), ]); } @@ -544,7 +544,6 @@ export class MarketplacesReindexService { ), ]; const assets = await this.mxApiService.getNftsByIdentifiers(assetsWithNoTagsIdentifiers, 0, 'fields=identifier,tags'); - const tags = assets.filter((a) => a.tags); for (let j = 0; j < assets?.length; j++) { auctions.filter((a) => a.identifier === assets[j].identifier).map((a) => (a.tags = assets[j]?.tags?.join(',') ?? '')); } From fd4e9f87d4294775270167f301a1063d1a0547dd Mon Sep 17 00:00:00 2001 From: danielailie Date: Mon, 13 Nov 2023 14:50:00 +0200 Subject: [PATCH 08/18] Remove tags update --- .../marketplaces-reindex.service.ts | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/modules/marketplaces/marketplaces-reindex.service.ts b/src/modules/marketplaces/marketplaces-reindex.service.ts index ce9d76634..6d1189802 100644 --- a/src/modules/marketplaces/marketplaces-reindex.service.ts +++ b/src/modules/marketplaces/marketplaces-reindex.service.ts @@ -480,21 +480,21 @@ export class MarketplacesReindexService { await this.auctionSetterService.saveBulkAuctionsOrUpdateAndFillId(inactiveAuctions); - let tags: TagEntity[] = []; - inactiveAuctions.map((auction) => { - const assetTags = auction.tags.split(','); - assetTags.map((assetTag) => { - if (assetTag !== '') { - tags.push( - new TagEntity({ - auctionId: auction.id, - tag: assetTag.trim().slice(0, constants.dbMaxTagLength), - auction: auction, - }), - ); - } - }); - }); + // let tags: TagEntity[] = []; + // inactiveAuctions.map((auction) => { + // const assetTags = auction.tags.split(','); + // assetTags.map((assetTag) => { + // if (assetTag !== '') { + // tags.push( + // new TagEntity({ + // auctionId: auction.id, + // tag: assetTag.trim().slice(0, constants.dbMaxTagLength), + // auction: auction, + // }), + // ); + // } + // }); + // }); // const saveTagsPromise = this.persistenceService.saveTagsOrIgnore(tags); From fb27890d01f899c5479ac068a08ba82010d45b13 Mon Sep 17 00:00:00 2001 From: danielailie Date: Tue, 14 Nov 2023 11:50:55 +0200 Subject: [PATCH 09/18] Add index on marketplace events table --- .../marketplaces/marketplace-events.entity.ts | 1 + ...9946-AddMarketplaceAddresTimestampIndex.ts | 14 +++++ .../reindex-auction-bid.handler.ts | 3 - .../reindex-auction-bought.handler.ts | 24 ++++---- .../reindex-auction-closed.handler.ts | 3 +- .../reindex-auction-ended.handler.ts | 11 +++- .../marketplaces-reindex.service.ts | 8 ++- .../models/MarketplaceReindexState.ts | 60 ++++++------------- 8 files changed, 61 insertions(+), 63 deletions(-) create mode 100644 src/db/migrations/1699949169946-AddMarketplaceAddresTimestampIndex.ts diff --git a/src/db/marketplaces/marketplace-events.entity.ts b/src/db/marketplaces/marketplace-events.entity.ts index 8882c6900..33110b013 100644 --- a/src/db/marketplaces/marketplace-events.entity.ts +++ b/src/db/marketplaces/marketplace-events.entity.ts @@ -4,6 +4,7 @@ import { Column, Entity, Index, Unique } from 'typeorm'; @Entity('marketplace_events') @Unique('MarketplaceEventsEntity_UQ_EVENT', ['txHash', 'eventOrder', 'isTx']) +@Index('idx_marketplaceAddress_timestamp', ['marketplaceAddress', 'timestamp']) export class MarketplaceEventsEntity extends BaseEntity { @Column({ length: 64 }) txHash: string; diff --git a/src/db/migrations/1699949169946-AddMarketplaceAddresTimestampIndex.ts b/src/db/migrations/1699949169946-AddMarketplaceAddresTimestampIndex.ts new file mode 100644 index 000000000..f5b8bb7a3 --- /dev/null +++ b/src/db/migrations/1699949169946-AddMarketplaceAddresTimestampIndex.ts @@ -0,0 +1,14 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class AddMarketplaceAddresTimestampIndex1699949169946 implements MigrationInterface { + name = 'AddMarketplaceAddresTimestampIndex1699949169946' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE INDEX \`idx_marketplaceAddress_timestamp\` ON \`marketplace_events\` (\`marketplaceAddress\`, \`timestamp\`)`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX \`idx_marketplaceAddress_timestamp\` ON \`marketplace_events\``); + } + +} diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bid.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bid.handler.ts index 43b5f0a3d..f5d279d0d 100644 --- a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bid.handler.ts +++ b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bid.handler.ts @@ -17,14 +17,11 @@ export class ReindexAuctionBidHandler { } let order = marketplaceReindexState.createOrder(auctionIndex, input, OrderStatusEnum.Active, paymentToken, paymentNonce); - if (order.priceAmount === marketplaceReindexState.auctions[auctionIndex].maxBid) { order.status = OrderStatusEnum.Bought; marketplaceReindexState.updateAuctionStatus(auctionIndex, input.blockHash, AuctionStatusEnum.Ended, input.timestamp); } marketplaceReindexState.updateOrderListForAuction(auctionIndex, order); - - marketplaceReindexState.orders.push(order); } } diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bought.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bought.handler.ts index d4dba7829..247e5de81 100644 --- a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bought.handler.ts +++ b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bought.handler.ts @@ -22,14 +22,11 @@ export class ReindexAuctionBoughtHandler { } const order = marketplaceReindexState.createOrder(auctionIndex, input, OrderStatusEnum.Bought, paymentToken, paymentNonce); - marketplaceReindexState.orders.push(order); + const auction = marketplaceReindexState.auctions[auctionIndex]; - const totalBought = this.getTotalBoughtTokensForAuction( - marketplaceReindexState.auctions[auctionIndex].id, - marketplaceReindexState.orders, - ); + const totalBought = this.getTotalBoughtTokensForAuction(auction.id, auction.orders); - if (marketplaceReindexState.auctions[auctionIndex].nrAuctionedTokens === totalBought) { + if (auction.nrAuctionedTokens === totalBought) { marketplaceReindexState.updateAuctionStatus(auctionIndex, input.blockHash, AuctionStatusEnum.Ended, input.timestamp); } @@ -38,11 +35,14 @@ export class ReindexAuctionBoughtHandler { private getTotalBoughtTokensForAuction(auctionId: number, orders: OrderEntity[]): number { let totalBought = 0; - orders - .filter((o) => o.auctionId === auctionId && o.status === OrderStatusEnum.Bought) - .forEach((o) => { - totalBought += parseInt(o.boughtTokensNo) ? parseInt(o.boughtTokensNo) : 1; - }); - return totalBought; + if (orders?.length) { + orders + .filter((o) => o.auctionId === auctionId && o.status === OrderStatusEnum.Bought) + .forEach((o) => { + totalBought += parseInt(o.boughtTokensNo) ? parseInt(o.boughtTokensNo) : 1; + }); + return totalBought; + } + return 0; } } diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-closed.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-closed.handler.ts index 826240c63..bc974ec93 100644 --- a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-closed.handler.ts +++ b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-closed.handler.ts @@ -22,7 +22,6 @@ export class ReindexAuctionClosedHandler { marketplaceReindexState.updateAuctionStatus(auctionIndex, input.blockHash, AuctionStatusEnum.Closed, input.timestamp); - marketplaceReindexState.setInactiveOrdersForAuction(marketplaceReindexState.auctions[auctionIndex].id, modifiedDate); - marketplaceReindexState.setInactiveOrdersForAuctionNew(auctionIndex, modifiedDate); + marketplaceReindexState.setInactiveOrdersForAuction(auctionIndex, modifiedDate); } } diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-ended.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-ended.handler.ts index d38ffb4a2..cad9d6b54 100644 --- a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-ended.handler.ts +++ b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-ended.handler.ts @@ -19,18 +19,23 @@ export class ReindexAuctionEndedHandler { } marketplaceReindexState.updateAuctionStatus(auctionIndex, input.blockHash, AuctionStatusEnum.Ended, input.timestamp); + const selectedAuction = marketplaceReindexState.auctions[auctionIndex]; const winnerOrderId = marketplaceReindexState.setAuctionOrderWinnerStatusAndReturnId( - marketplaceReindexState.auctions[auctionIndex].id, + selectedAuction.id, OrderStatusEnum.Bought, modifiedDate, ); if (winnerOrderId !== -1) { - marketplaceReindexState.setInactiveOrdersForAuction(marketplaceReindexState.auctions[auctionIndex].id, modifiedDate, winnerOrderId); + marketplaceReindexState.setInactiveOrdersForAuction(selectedAuction.id, modifiedDate, winnerOrderId); } else if (input.currentBid !== '0') { const order = marketplaceReindexState.createOrder(auctionIndex, input, OrderStatusEnum.Bought, paymentToken); - marketplaceReindexState.orders.push(order); + if (marketplaceReindexState.auctions[auctionIndex].orders) { + marketplaceReindexState.auctions[auctionIndex].orders.push(order); + } else { + marketplaceReindexState.auctions[auctionIndex].orders = [order]; + } } } } diff --git a/src/modules/marketplaces/marketplaces-reindex.service.ts b/src/modules/marketplaces/marketplaces-reindex.service.ts index 6d1189802..37d8608ee 100644 --- a/src/modules/marketplaces/marketplaces-reindex.service.ts +++ b/src/modules/marketplaces/marketplaces-reindex.service.ts @@ -24,12 +24,11 @@ import { MarketplaceReindexState } from './models/MarketplaceReindexState'; import { MxApiService } from 'src/common'; import { MarketplaceReindexDataArgs } from './models/MarketplaceReindexDataArgs'; import { ELRONDNFTSWAP_KEY } from 'src/utils/constants'; -import { DateUtils } from 'src/utils/date-utils'; import { Locker } from '@multiversx/sdk-nestjs-common'; -import { TagEntity } from 'src/db/auctions/tags.entity'; import { MarketplaceTypeEnum } from './models/MarketplaceType.enum'; import { OrdersService } from '../orders/order.service'; import { Token } from '../usdPrice/Token.model'; +import { CpuProfiler } from '@multiversx/sdk-nestjs-monitoring'; @Injectable() export class MarketplacesReindexService { @@ -120,13 +119,18 @@ export class MarketplacesReindexService { nextBatchPromise = this.persistenceService.getMarketplaceEventsAsc(marketplaceReindexStates[0].marketplace.address, afterTimestamp); do { + const cpu = new CpuProfiler(); let batch = await nextBatchPromise; console.log({ batch: batch.length, afterTimestamp }); if (!batch || batch.length === 0) { break; } + cpu.stop('batch'); + + const getSlicedBatchAndNewestTimestampIfPartialEventsSet = new CpuProfiler(); [batch, afterTimestamp] = this.getSlicedBatchAndNewestTimestampIfPartialEventsSet(batch, input.beforeTimestamp); + getSlicedBatchAndNewestTimestampIfPartialEventsSet.stop('getSlicedBatchAndNewestTimestampIfPartialEventsSet'); nextBatchPromise = this.persistenceService.getMarketplaceEventsAsc(marketplaceReindexStates[0].marketplace.address, afterTimestamp); processInNextBatch = await this.processEventsBatchAndReturnUnprocessedEvents( diff --git a/src/modules/marketplaces/models/MarketplaceReindexState.ts b/src/modules/marketplaces/models/MarketplaceReindexState.ts index 390bafeeb..7ff6b95c5 100644 --- a/src/modules/marketplaces/models/MarketplaceReindexState.ts +++ b/src/modules/marketplaces/models/MarketplaceReindexState.ts @@ -83,55 +83,33 @@ export class MarketplaceReindexState { } setAuctionOrderWinnerStatusAndReturnId(auctionId: number, status: OrderStatusEnum, modifiedDate?: Date): number { - const bids = this.orders - .filter((o) => o.auctionId === auctionId && o.status === OrderStatusEnum.Active) - .map((o) => new BigNumber(o.priceAmount)); - - if (bids.length) { - const maxBid = BigNumber.max(...bids); - const winnerOrderIndex = this.orders.findIndex( - (o) => o.auctionId === auctionId && o.status === OrderStatusEnum.Active && o.priceAmount === maxBid.toString(), - ); - this.orders[winnerOrderIndex].status = status; - if (modifiedDate) { - this.orders[winnerOrderIndex].modifiedDate = modifiedDate; - } - return this.orders[winnerOrderIndex].id; + const selectedAuction = this.auctions[auctionId]; + if (!selectedAuction) { + return -1; } - return -1; - } - setAuctionOrderWinnerStatusAndeturnId(auctionId: number, status: OrderStatusEnum, modifiedDate?: Date): number { - const bids = this.orders - .filter((o) => o.auctionId === auctionId && o.status === OrderStatusEnum.Active) - .map((o) => new BigNumber(o.priceAmount)); + const activeOrders = selectedAuction.orders.filter((o) => o.status === OrderStatusEnum.Active); + if (activeOrders.length === 0) { + return -1; + } - if (bids.length) { - const maxBid = BigNumber.max(...bids); - const winnerOrderIndex = this.orders.findIndex( - (o) => o.auctionId === auctionId && o.status === OrderStatusEnum.Active && o.priceAmount === maxBid.toString(), - ); - this.orders[winnerOrderIndex].status = status; - if (modifiedDate) { - this.orders[winnerOrderIndex].modifiedDate = modifiedDate; - } - return this.orders[winnerOrderIndex].id; + const maxBid = BigNumber.max(...activeOrders.map((o) => new BigNumber(o.priceAmount))); + const winnerOrder = activeOrders.find((o) => new BigNumber(o.priceAmount).isEqualTo(maxBid)); + if (!winnerOrder) { + return -1; } - return -1; - } - setInactiveOrdersForAuction(auctionId: number, modifiedDate: Date, exceptWinnerId?: number): void { - this.orders - ?.filter((o) => o.auctionId === auctionId && o.status === OrderStatusEnum.Active && o.id !== exceptWinnerId) - ?.map((o) => { - o.status = OrderStatusEnum.Inactive; - o.modifiedDate = modifiedDate; - }); + winnerOrder.status = status; + if (modifiedDate) { + winnerOrder.modifiedDate = modifiedDate; + } + + return winnerOrder.id; } - setInactiveOrdersForAuctionNew(auctionId: number, modifiedDate: Date, exceptWinnerId?: number): void { + setInactiveOrdersForAuction(auctionId: number, modifiedDate: Date, exceptWinnerId?: number): void { this.auctions[auctionId]?.orders - ?.filter((o) => o.auctionId === auctionId && o.status === OrderStatusEnum.Active && o.id !== exceptWinnerId) + ?.filter((o) => o.status === OrderStatusEnum.Active && o.id !== exceptWinnerId) ?.map((o) => { o.status = OrderStatusEnum.Inactive; o.modifiedDate = modifiedDate; From a1d47cdf6b6b80536f2e5c195e3048bdb1c24585 Mon Sep 17 00:00:00 2001 From: danielailie Date: Wed, 15 Nov 2023 11:22:52 +0200 Subject: [PATCH 10/18] performance improvments --- .../reindex-auction-bought.handler.ts | 8 ++- .../reindex-auction-price-updated.handler.ts | 30 ++++----- .../models/MarketplaceReindexState.ts | 62 +++++++++++-------- 3 files changed, 55 insertions(+), 45 deletions(-) diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bought.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bought.handler.ts index 247e5de81..eb5f15a95 100644 --- a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bought.handler.ts +++ b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bought.handler.ts @@ -24,10 +24,12 @@ export class ReindexAuctionBoughtHandler { const order = marketplaceReindexState.createOrder(auctionIndex, input, OrderStatusEnum.Bought, paymentToken, paymentNonce); const auction = marketplaceReindexState.auctions[auctionIndex]; - const totalBought = this.getTotalBoughtTokensForAuction(auction.id, auction.orders); + if (auction.nrAuctionedTokens > 1) { + const totalBought = this.getTotalBoughtTokensForAuction(auction.id, auction.orders); - if (auction.nrAuctionedTokens === totalBought) { - marketplaceReindexState.updateAuctionStatus(auctionIndex, input.blockHash, AuctionStatusEnum.Ended, input.timestamp); + if (auction.nrAuctionedTokens === totalBought) { + marketplaceReindexState.updateAuctionStatus(auctionIndex, input.blockHash, AuctionStatusEnum.Ended, input.timestamp); + } } marketplaceReindexState.updateOrderListForAuction(auctionIndex, order); diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-price-updated.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-price-updated.handler.ts index 8b2f513a1..8320e0975 100644 --- a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-price-updated.handler.ts +++ b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-price-updated.handler.ts @@ -11,37 +11,33 @@ export class ReindexAuctionPriceUpdatedHandler { handle(marketplaceReindexState: MarketplaceReindexState, input: AuctionPriceUpdatedSummary, decimals: number): void { const auctionIndex = marketplaceReindexState.getAuctionIndexByAuctionId(input.auctionId); - const modifiedDate = DateUtils.getUtcDateFromTimestamp(input.timestamp); + if (auctionIndex === -1) { return; } - marketplaceReindexState.auctions[auctionIndex].blockHash = marketplaceReindexState.auctions[auctionIndex].blockHash ?? input.blockHash; - marketplaceReindexState.auctions[auctionIndex].modifiedDate = modifiedDate; + const modifiedDate = DateUtils.getUtcDateFromTimestamp(input.timestamp); + const auction = marketplaceReindexState.auctions[auctionIndex]; - marketplaceReindexState.auctions[auctionIndex].minBid = input.minBid; - marketplaceReindexState.auctions[auctionIndex].minBidDenominated = Math.min( - BigNumberUtils.denominateAmount(input.minBid, decimals), - constants.dbMaxDenominatedValue, - ); + auction.blockHash = auction.blockHash ?? input.blockHash; + auction.modifiedDate = modifiedDate; + auction.minBid = input.minBid; + auction.minBidDenominated = Math.min(BigNumberUtils.denominateAmount(input.minBid, decimals), constants.dbMaxDenominatedValue); if (input.maxBid) { - marketplaceReindexState.auctions[auctionIndex].maxBid = input.maxBid; - marketplaceReindexState.auctions[auctionIndex].maxBidDenominated = Math.min( - BigNumberUtils.denominateAmount(input.maxBid, decimals), - constants.dbMaxDenominatedValue, - ); + auction.maxBid = input.maxBid; + auction.maxBidDenominated = Math.min(BigNumberUtils.denominateAmount(input.maxBid, decimals), constants.dbMaxDenominatedValue); } else { - marketplaceReindexState.auctions[auctionIndex].maxBid = input.minBid; - marketplaceReindexState.auctions[auctionIndex].maxBidDenominated = marketplaceReindexState.auctions[auctionIndex].minBidDenominated; + auction.maxBid = auction.minBid; + auction.maxBidDenominated = auction.minBidDenominated; } if (input.paymentToken) { - marketplaceReindexState.auctions[auctionIndex].paymentToken = input.paymentToken; + auction.paymentToken = input.paymentToken; } if (input.itemsCount) { - marketplaceReindexState.auctions[auctionIndex].nrAuctionedTokens = input.itemsCount; + auction.nrAuctionedTokens = input.itemsCount; } } } diff --git a/src/modules/marketplaces/models/MarketplaceReindexState.ts b/src/modules/marketplaces/models/MarketplaceReindexState.ts index 7ff6b95c5..7f801c76a 100644 --- a/src/modules/marketplaces/models/MarketplaceReindexState.ts +++ b/src/modules/marketplaces/models/MarketplaceReindexState.ts @@ -46,11 +46,12 @@ export class MarketplaceReindexState { } getAuctionIndexByAuctionId(auctionId: number): number { - return this.auctions.findIndex((a) => a.marketplaceAuctionId === auctionId); - } - - getAuctionByAuctionId(auctionId: number): number { - return this.auctions.findIndex((a) => a.marketplaceAuctionId === auctionId); + for (let i = 0; i < this.auctions.length; i++) { + if (this.auctions[i].marketplaceAuctionId === auctionId) { + return i; + } + } + return -1; } getAuctionIndexByIdentifier(identifier: string): number { @@ -108,12 +109,16 @@ export class MarketplaceReindexState { } setInactiveOrdersForAuction(auctionId: number, modifiedDate: Date, exceptWinnerId?: number): void { - this.auctions[auctionId]?.orders - ?.filter((o) => o.status === OrderStatusEnum.Active && o.id !== exceptWinnerId) - ?.map((o) => { - o.status = OrderStatusEnum.Inactive; - o.modifiedDate = modifiedDate; - }); + const auction = this.auctions[auctionId]; + + if (auction && auction.orders) { + for (const order of auction.orders) { + if (order.status === OrderStatusEnum.Active && order.id !== exceptWinnerId) { + order.status = OrderStatusEnum.Inactive; + order.modifiedDate = modifiedDate; + } + } + } } setStateItemsToExpiredIfOlderThanTimestamp(timestamp: number): void { @@ -181,18 +186,26 @@ export class MarketplaceReindexState { } public updateOrderListForAuction(auctionIndex: number, order: OrderEntity) { - this.auctions[auctionIndex].orders = this.auctions[auctionIndex].orders - ? [...this.auctions[auctionIndex].orders?.map((order) => ({ ...order, status: OrderStatusEnum.Inactive })), order] - : [order]; - } + const existingOrders = this.auctions[auctionIndex].orders || []; - public updateAuctionStatus(auctionIndex: number, blockHash: string, status: AuctionStatusEnum, timestamp: number) { - this.auctions[auctionIndex] = { - ...this.auctions[auctionIndex], - status: status, - blockHash: this.auctions[auctionIndex].blockHash ?? blockHash, - modifiedDate: DateUtils.getUtcDateFromTimestamp(timestamp), - }; + this.auctions[auctionIndex].orders = [ + ...existingOrders.map((existingOrder) => ({ ...existingOrder, status: OrderStatusEnum.Inactive })), + order, + ]; + } + public updateAuctionStatus(auctionIndex: number, blockHash: string, status: AuctionStatusEnum, timestamp: number): void { + const currentAuction = this.auctions[auctionIndex]; + + if (currentAuction) { + const updatedAuction = { + ...currentAuction, + status, + blockHash: currentAuction.blockHash ?? blockHash, + modifiedDate: DateUtils.getUtcDateFromTimestamp(timestamp), + }; + + this.auctions[auctionIndex] = updatedAuction; + } } deleteAuctionIfDuplicates(marketplaceAuctionId: number) { @@ -201,10 +214,9 @@ export class MarketplaceReindexState { } let index; - do { - index = this.auctions.findIndex((a) => a.marketplaceAuctionId === marketplaceAuctionId); + while ((index = this.auctions.findIndex((a) => a.marketplaceAuctionId === marketplaceAuctionId)) !== -1) { this.auctions.splice(index, 1); - } while (index !== -1); + } } deleteOfferIfDuplicates(marketplaceOfferId: number) { From dcf1e8c1b59e15442ce387e85443f955993a0a0a Mon Sep 17 00:00:00 2001 From: danielailie Date: Thu, 16 Nov 2023 15:34:49 +0200 Subject: [PATCH 11/18] Use map instead of array --- src/db/auctions/auctions.repository.ts | 120 ++++++++------ .../auctions/auctions-setter.service.ts | 8 +- .../reindex-auction-bid.handler.ts | 12 +- .../reindex-auction-bought.handler.ts | 15 +- .../reindex-auction-closed.handler.ts | 12 +- .../reindex-auction-ended.handler.ts | 23 ++- .../reindex-auction-price-updated.handler.ts | 5 +- .../reindex-auction-started.handler.ts | 6 +- .../reindex-auction-updated.handler.ts | 28 ++-- .../reindex-global-offer-accepted.handler.ts | 8 +- .../reindex-offer-accepted.handler.ts | 12 +- .../marketplaces-reindex.service.ts | 91 ++++------- .../models/MarketplaceReindexState.ts | 153 +++--------------- 13 files changed, 184 insertions(+), 309 deletions(-) diff --git a/src/db/auctions/auctions.repository.ts b/src/db/auctions/auctions.repository.ts index 768b3dfe5..536d422c8 100644 --- a/src/db/auctions/auctions.repository.ts +++ b/src/db/auctions/auctions.repository.ts @@ -32,6 +32,8 @@ import { getLowestAuctionForIdentifiersAndMarketplace, getOnSaleAssetsCountForCollection, } from './sql.queries'; +import { CpuProfiler } from '@multiversx/sdk-nestjs-monitoring'; +import { OrderEntity } from '../orders'; @Injectable() export class AuctionsRepository { @@ -434,58 +436,78 @@ export class AuctionsRepository { } async saveBulkAuctionsOrUpdateAndFillId(auctions: AuctionEntity[]): Promise { + const batchSize = 1000; if (auctions.length === 0) { return; } - const response = await this.auctionsRepository.upsert(auctions, { conflictPaths: ['marketplaceAuctionId', 'marketplaceKey'] }); - - // const saveOrUpdateResponse = await this.auctionsRepository - // .createQueryBuilder() - // .insert() - // .into('auctions') - // .values(auctions) - // .orUpdate({ - // overwrite: [ - // 'creationDate', - // 'modifiedDate', - // 'collection', - // 'nrAuctionedTokens', - // 'identifier', - // 'nonce', - // 'status', - // 'type', - // 'paymentToken', - // 'paymentNonce', - // 'ownerAddress', - // 'minBidDiff', - // 'minBid', - // 'minBidDenominated', - // 'maxBid', - // 'maxBidDenominated', - // 'startDate', - // 'endDate', - // 'tags', - // 'blockHash', - // ], - // conflict_target: ['marketplaceAuctionId', 'marketplaceKey'], - // }) - // .updateEntity(false) - // .execute(); - // if (saveOrUpdateResponse.identifiers.length === 0 || auctions.findIndex((a) => a.id === undefined) !== -1) { - // const dbAuctions = await this.getBulkAuctionsByMarketplaceAndAuctionIds( - // auctions?.[0]?.marketplaceKey, - // auctions?.map((a) => a.marketplaceAuctionId), - // ); - // for (let i = 0; i < dbAuctions.length; i++) { - // const auctionIndex = auctions.findIndex((a) => a.marketplaceAuctionId === dbAuctions[i].marketplaceAuctionId); - // auctions[auctionIndex].id = dbAuctions[i].id; - // } - // } - // if (auctions.findIndex((a) => a.id === undefined) !== -1) { - // const wtf = auctions.filter((a) => a.id === undefined).map((a) => a.marketplaceAuctionId); - // const duplicate = auctions.filter((a) => a.marketplaceAuctionId === 31434); - // throw new Error(`oooppps ${JSON.stringify(duplicate)}`); - // } + + const connection = this.auctionsRepository.manager.connection; + + await connection.transaction(async (transactionalEntityManager) => { + const queryBuilder = transactionalEntityManager.createQueryBuilder().from(AuctionEntity, 'auction'); + + for (let i = 0; i < auctions.length; i += batchSize) { + const currentQueryBuilder = queryBuilder.clone(); + const batch = auctions.slice(i, i + batchSize); + console.log('Processing batch number', i); + + const cpu = new CpuProfiler(); + for (const item of batch) { + currentQueryBuilder.insert().values(item).onConflict(`("marketplaceKey", "marketplaceAuctionId") DO UPDATE SET + "collection" = EXCLUDED."collection", + "nrAuctionedTokens" = EXCLUDED."nrAuctionedTokens", + "identifier" = EXCLUDED."identifier", + "nonce" = EXCLUDED."nonce", + "status" = EXCLUDED."status", + "type" = EXCLUDED."type", + "paymentToken" = EXCLUDED."paymentToken", + "paymentNonce" = EXCLUDED."paymentNonce", + "ownerAddress" = EXCLUDED."ownerAddress", + "minBidDiff" = EXCLUDED."minBidDiff", + "minBid" = EXCLUDED."minBid", + "minBidDenominated" = EXCLUDED."minBidDenominated", + "maxBid" = EXCLUDED."maxBid", + "maxBidDenominated" = EXCLUDED."maxBidDenominated", + "startDate" = EXCLUDED."startDate", + "endDate" = EXCLUDED."endDate", + "tags" = EXCLUDED."tags", + "blockHash" = EXCLUDED."blockHash" + `); + + // Include related orders + if (item.orders && item.orders.length > 0) { + for (const order of item.orders) { + currentQueryBuilder.insert().into(OrderEntity).values({ + priceToken: order.priceToken, + priceAmount: order.priceAmount, + priceAmountDenominated: order.priceAmountDenominated, + priceNonce: order.priceNonce, + status: order.status, + ownerAddress: order.ownerAddress, + boughtTokensNo: order.boughtTokensNo, + auctionId: order.auctionId, + blockHash: order.blockHash, + marketplaceKey: order.marketplaceKey, + }).onConflict(`("auctionId", "marketplaceKey") DO UPDATE SET + "priceToken" = EXCLUDED."priceToken", + "priceAmount" = EXCLUDED."priceAmount", + "priceAmountDenominated" = EXCLUDED."priceAmountDenominated", + "priceNonce" = EXCLUDED."priceNonce", + "status" = EXCLUDED."status", + "ownerAddress" = EXCLUDED."ownerAddress", + "boughtTokensNo" = EXCLUDED."boughtTokensNo", + "blockHash" = EXCLUDED."blockHash" + `); + } + } + } + + await currentQueryBuilder.execute(); + cpu.stop(`batch ${i}`); + } + + console.log('Bulk insert or update completed successfully.'); + }); } async rollbackAuctionAndOrdersByHash(blockHash: string): Promise { diff --git a/src/modules/auctions/auctions-setter.service.ts b/src/modules/auctions/auctions-setter.service.ts index 81455b3a6..fc0ff4201 100644 --- a/src/modules/auctions/auctions-setter.service.ts +++ b/src/modules/auctions/auctions-setter.service.ts @@ -89,12 +89,8 @@ export class AuctionsSetterService { } } - async saveBulkAuctionsOrUpdateAndFillId( - auctions: AuctionEntity[], - ): Promise { - return await this.persistenceService.saveBulkAuctionsOrUpdateAndFillId( - auctions, - ); + async saveBulkAuctionsOrUpdateAndFillId(auctions: AuctionEntity[]): Promise { + return await this.persistenceService.saveBulkAuctionsOrUpdateAndFillId(auctions); } async saveAuctionEntity(auctionEntity: AuctionEntity, assetTags: string[]): Promise { diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bid.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bid.handler.ts index f5d279d0d..00bbff34e 100644 --- a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bid.handler.ts +++ b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bid.handler.ts @@ -10,18 +10,18 @@ export class ReindexAuctionBidHandler { constructor() {} handle(marketplaceReindexState: MarketplaceReindexState, input: AuctionBidSummary, paymentToken: Token, paymentNonce: number): void { - const auctionIndex = marketplaceReindexState.getAuctionIndexByAuctionId(input.auctionId); + const auction = marketplaceReindexState.auctionMap.get(input.auctionId); - if (auctionIndex === -1) { + if (!auction) { return; } - let order = marketplaceReindexState.createOrder(auctionIndex, input, OrderStatusEnum.Active, paymentToken, paymentNonce); - if (order.priceAmount === marketplaceReindexState.auctions[auctionIndex].maxBid) { + let order = marketplaceReindexState.createOrder(auction, input, OrderStatusEnum.Active, paymentToken, paymentNonce); + if (order.priceAmount === auction.maxBid) { order.status = OrderStatusEnum.Bought; - marketplaceReindexState.updateAuctionStatus(auctionIndex, input.blockHash, AuctionStatusEnum.Ended, input.timestamp); + marketplaceReindexState.updateAuctionStatus(auction, input.blockHash, AuctionStatusEnum.Ended, input.timestamp); } - marketplaceReindexState.updateOrderListForAuction(auctionIndex, order); + marketplaceReindexState.updateOrderListForAuction(auction, order); } } diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bought.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bought.handler.ts index eb5f15a95..f4848fd4c 100644 --- a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bought.handler.ts +++ b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bought.handler.ts @@ -12,27 +12,26 @@ export class ReindexAuctionBoughtHandler { constructor() {} handle(marketplaceReindexState: MarketplaceReindexState, input: AuctionBuySummary, paymentToken: Token, paymentNonce: number): void { - const auctionIndex = + const auction = marketplaceReindexState.marketplace.key !== ELRONDNFTSWAP_KEY - ? marketplaceReindexState.getAuctionIndexByAuctionId(input.auctionId) - : marketplaceReindexState.getAuctionIndexByIdentifier(input.identifier); + ? marketplaceReindexState.auctionMap.get(input.auctionId) + : marketplaceReindexState.auctionMap.get(input.auctionId); //de scris - if (auctionIndex === -1) { + if (!auction) { return; } - const order = marketplaceReindexState.createOrder(auctionIndex, input, OrderStatusEnum.Bought, paymentToken, paymentNonce); - const auction = marketplaceReindexState.auctions[auctionIndex]; + const order = marketplaceReindexState.createOrder(auction, input, OrderStatusEnum.Bought, paymentToken, paymentNonce); if (auction.nrAuctionedTokens > 1) { const totalBought = this.getTotalBoughtTokensForAuction(auction.id, auction.orders); if (auction.nrAuctionedTokens === totalBought) { - marketplaceReindexState.updateAuctionStatus(auctionIndex, input.blockHash, AuctionStatusEnum.Ended, input.timestamp); + marketplaceReindexState.updateAuctionStatus(auction, input.blockHash, AuctionStatusEnum.Ended, input.timestamp); } } - marketplaceReindexState.updateOrderListForAuction(auctionIndex, order); + marketplaceReindexState.updateOrderListForAuction(auction, order); } private getTotalBoughtTokensForAuction(auctionId: number, orders: OrderEntity[]): number { diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-closed.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-closed.handler.ts index bc974ec93..635791513 100644 --- a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-closed.handler.ts +++ b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-closed.handler.ts @@ -10,18 +10,18 @@ export class ReindexAuctionClosedHandler { constructor() {} handle(marketplaceReindexState: MarketplaceReindexState, input: ReindexAuctionClosedSummary): void { - const auctionIndex = + const auction = marketplaceReindexState.marketplace.key !== ELRONDNFTSWAP_KEY - ? marketplaceReindexState.getAuctionIndexByAuctionId(input.auctionId) - : marketplaceReindexState.getAuctionIndexByIdentifier(input.identifier); + ? marketplaceReindexState.auctionMap.get(input.auctionId) + : marketplaceReindexState.auctionMap.get(input.auctionId); //de scris const modifiedDate = DateUtils.getUtcDateFromTimestamp(input.timestamp); - if (auctionIndex === -1) { + if (!auction) { return; } - marketplaceReindexState.updateAuctionStatus(auctionIndex, input.blockHash, AuctionStatusEnum.Closed, input.timestamp); + marketplaceReindexState.updateAuctionStatus(auction, input.blockHash, AuctionStatusEnum.Closed, input.timestamp); - marketplaceReindexState.setInactiveOrdersForAuction(auctionIndex, modifiedDate); + marketplaceReindexState.setInactiveOrdersForAuction(auction, modifiedDate); } } diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-ended.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-ended.handler.ts index cad9d6b54..01be73bbe 100644 --- a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-ended.handler.ts +++ b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-ended.handler.ts @@ -11,30 +11,25 @@ export class ReindexAuctionEndedHandler { constructor() {} handle(marketplaceReindexState: MarketplaceReindexState, input: AuctionEndedSummary, paymentToken: Token): void { - const auctionIndex = marketplaceReindexState.getAuctionIndexByAuctionId(input.auctionId); + const auction = marketplaceReindexState.auctionMap.get(input.auctionId); const modifiedDate = DateUtils.getUtcDateFromTimestamp(input.timestamp); - if (auctionIndex === -1) { + if (!auction) { return; } - marketplaceReindexState.updateAuctionStatus(auctionIndex, input.blockHash, AuctionStatusEnum.Ended, input.timestamp); - const selectedAuction = marketplaceReindexState.auctions[auctionIndex]; + marketplaceReindexState.updateAuctionStatus(auction, input.blockHash, AuctionStatusEnum.Ended, input.timestamp); - const winnerOrderId = marketplaceReindexState.setAuctionOrderWinnerStatusAndReturnId( - selectedAuction.id, - OrderStatusEnum.Bought, - modifiedDate, - ); + const winnerOrderId = marketplaceReindexState.setAuctionOrderWinnerStatusAndReturnId(auction, OrderStatusEnum.Bought, modifiedDate); if (winnerOrderId !== -1) { - marketplaceReindexState.setInactiveOrdersForAuction(selectedAuction.id, modifiedDate, winnerOrderId); + marketplaceReindexState.setInactiveOrdersForAuction(auction, modifiedDate, winnerOrderId); } else if (input.currentBid !== '0') { - const order = marketplaceReindexState.createOrder(auctionIndex, input, OrderStatusEnum.Bought, paymentToken); - if (marketplaceReindexState.auctions[auctionIndex].orders) { - marketplaceReindexState.auctions[auctionIndex].orders.push(order); + const order = marketplaceReindexState.createOrder(auction, input, OrderStatusEnum.Bought, paymentToken); + if (auction.orders) { + marketplaceReindexState.auctionMap.get(auction.marketplaceAuctionId).orders.push(order); } else { - marketplaceReindexState.auctions[auctionIndex].orders = [order]; + marketplaceReindexState.auctionMap.get(auction.marketplaceAuctionId).orders = [order]; } } } diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-price-updated.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-price-updated.handler.ts index 8320e0975..a438cabdd 100644 --- a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-price-updated.handler.ts +++ b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-price-updated.handler.ts @@ -10,14 +10,13 @@ export class ReindexAuctionPriceUpdatedHandler { constructor() {} handle(marketplaceReindexState: MarketplaceReindexState, input: AuctionPriceUpdatedSummary, decimals: number): void { - const auctionIndex = marketplaceReindexState.getAuctionIndexByAuctionId(input.auctionId); + const auction = marketplaceReindexState.auctionMap.get(input.auctionId); - if (auctionIndex === -1) { + if (!auction) { return; } const modifiedDate = DateUtils.getUtcDateFromTimestamp(input.timestamp); - const auction = marketplaceReindexState.auctions[auctionIndex]; auction.blockHash = auction.blockHash ?? input.blockHash; auction.modifiedDate = modifiedDate; diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-started.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-started.handler.ts index 3f58fb6eb..d832da007 100644 --- a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-started.handler.ts +++ b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-started.handler.ts @@ -23,13 +23,11 @@ export class ReindexAuctionStartedHandler { const minBidDenominated = BigNumberUtils.denominateAmount(input.minBid, paymentToken.decimals); const maxBidDenominated = BigNumberUtils.denominateAmount(input.maxBid !== 'NaN' ? input.maxBid : '0', paymentToken.decimals); - marketplaceReindexState.deleteAuctionIfDuplicates(input.auctionId); - const auction = new AuctionEntity({ creationDate: modifiedDate, modifiedDate, id: marketplaceReindexState.getNewAuctionId(), - marketplaceAuctionId: input.auctionId !== 0 ? input.auctionId : marketplaceReindexState.auctions.length + 1, + marketplaceAuctionId: input.auctionId !== 0 ? input.auctionId : marketplaceReindexState.auctionMap.size + 1, identifier: input.identifier, collection: input.collection, nonce: nonce, @@ -51,6 +49,6 @@ export class ReindexAuctionStartedHandler { marketplaceKey: marketplaceReindexState.marketplace.key, }); - marketplaceReindexState.auctions.push(auction); + marketplaceReindexState.auctionMap.set(auction.marketplaceAuctionId, auction); } } diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-updated.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-updated.handler.ts index 8b0350795..58de79153 100644 --- a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-updated.handler.ts +++ b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-updated.handler.ts @@ -11,34 +11,28 @@ export class ReindexAuctionUpdatedHandler { constructor() {} handle(marketplaceReindexState: MarketplaceReindexState, input: AuctionUpdatedSummary, decimals: number): void { - const auctionIndex = marketplaceReindexState.getAuctionIndexByAuctionId(input.auctionId); + const auction = marketplaceReindexState.auctionMap.get(input.auctionId); const modifiedDate = DateUtils.getUtcDateFromTimestamp(input.timestamp); - if (auctionIndex === -1) { + if (!auction) { return; } - marketplaceReindexState.auctions[auctionIndex].blockHash = marketplaceReindexState.auctions[auctionIndex].blockHash ?? input.blockHash; - marketplaceReindexState.auctions[auctionIndex].modifiedDate = modifiedDate; + auction.blockHash = auction.blockHash ?? input.blockHash; + auction.modifiedDate = modifiedDate; - marketplaceReindexState.auctions[auctionIndex].minBid = input.minBid; - marketplaceReindexState.auctions[auctionIndex].minBidDenominated = Math.min( - BigNumberUtils.denominateAmount(input.minBid, decimals), - constants.dbMaxDenominatedValue, - ); - marketplaceReindexState.auctions[auctionIndex].maxBid = input.minBid; - marketplaceReindexState.auctions[auctionIndex].maxBidDenominated = Math.min( - marketplaceReindexState.auctions[auctionIndex].minBidDenominated, - constants.dbMaxDenominatedValue, - ); + auction.minBid = input.minBid; + auction.minBidDenominated = Math.min(BigNumberUtils.denominateAmount(input.minBid, decimals), constants.dbMaxDenominatedValue); + auction.maxBid = input.minBid; + auction.maxBidDenominated = Math.min(auction.minBidDenominated, constants.dbMaxDenominatedValue); if (input.paymentToken) { - marketplaceReindexState.auctions[auctionIndex].paymentToken = input.paymentToken; - marketplaceReindexState.auctions[auctionIndex].paymentNonce = BinaryUtils.hexToNumber(input.paymentNonce); + auction.paymentToken = input.paymentToken; + auction.paymentNonce = BinaryUtils.hexToNumber(input.paymentNonce); } if (input.deadline > 0) { - marketplaceReindexState.auctions[auctionIndex].endDate = input.deadline; + auction.endDate = input.deadline; } } } diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-global-offer-accepted.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-global-offer-accepted.handler.ts index afd68cf8b..6bf8132b1 100644 --- a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-global-offer-accepted.handler.ts +++ b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-global-offer-accepted.handler.ts @@ -10,13 +10,13 @@ export class ReindexGlobalOfferAcceptedHandler { handle(marketplaceReindexState: MarketplaceReindexState, input: GlobalOfferAcceptedSummary): void { const modifiedDate = DateUtils.getUtcDateFromTimestamp(input.timestamp); - const auctionIndex = marketplaceReindexState.getAuctionIndexByAuctionId(input.auctionId); + const auction = marketplaceReindexState.auctionMap.get(input.auctionId); - if (auctionIndex === -1) { + if (!auction) { return; } - marketplaceReindexState.auctions[auctionIndex].status = AuctionStatusEnum.Closed; - marketplaceReindexState.auctions[auctionIndex].modifiedDate = modifiedDate; + auction.status = AuctionStatusEnum.Closed; + auction.modifiedDate = modifiedDate; } } diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-offer-accepted.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-offer-accepted.handler.ts index 048d1799b..711952043 100644 --- a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-offer-accepted.handler.ts +++ b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-offer-accepted.handler.ts @@ -13,10 +13,10 @@ export class ReindexOfferAcceptedHandler { handle(marketplaceReindexState: MarketplaceReindexState, input: OfferAcceptedSummary): void { const modifiedDate = DateUtils.getUtcDateFromTimestamp(input.timestamp); const offerIndex = marketplaceReindexState.getOfferIndexByOfferId(input.offerId); - const auctionIndex = + const auction = marketplaceReindexState.marketplace.key !== ELRONDNFTSWAP_KEY - ? marketplaceReindexState.getAuctionIndexByAuctionId(input.auctionId) - : marketplaceReindexState.getAuctionIndexByIdentifier(input.identifier); + ? marketplaceReindexState.auctionMap.get(input.auctionId) + : marketplaceReindexState.auctionMap.get(input.auctionId); //de scris if (offerIndex !== -1) { marketplaceReindexState.offers[offerIndex].status = OfferStatusEnum.Accepted; @@ -24,9 +24,9 @@ export class ReindexOfferAcceptedHandler { return; } - if (auctionIndex !== -1) { - marketplaceReindexState.auctions[auctionIndex].status = AuctionStatusEnum.Closed; - marketplaceReindexState.auctions[auctionIndex].modifiedDate = modifiedDate; + if (auction) { + auction.status = AuctionStatusEnum.Closed; + auction.modifiedDate = modifiedDate; } } } diff --git a/src/modules/marketplaces/marketplaces-reindex.service.ts b/src/modules/marketplaces/marketplaces-reindex.service.ts index 37d8608ee..8b00d9b19 100644 --- a/src/modules/marketplaces/marketplaces-reindex.service.ts +++ b/src/modules/marketplaces/marketplaces-reindex.service.ts @@ -29,6 +29,7 @@ import { MarketplaceTypeEnum } from './models/MarketplaceType.enum'; import { OrdersService } from '../orders/order.service'; import { Token } from '../usdPrice/Token.model'; import { CpuProfiler } from '@multiversx/sdk-nestjs-monitoring'; +import { TagEntity } from 'src/db/auctions/tags.entity'; @Injectable() export class MarketplacesReindexService { @@ -66,6 +67,7 @@ export class MarketplacesReindexService { this.logger.log(`Reindexing marketplace data/state for ${input.marketplaceAddress} ended`); } catch (error) { + console.log(error); this.logger.error('An error occurred while reindexing marketplace data', { path: `${MarketplacesReindexService.name}.${this.reindexMarketplaceData.name}`, marketplaceAddress: input.marketplaceAddress, @@ -138,8 +140,7 @@ export class MarketplacesReindexService { processInNextBatch.concat(batch), ); console.log({ - auctions: marketplaceReindexStates[0].auctions.length, - orders: marketplaceReindexStates[0].orders.length, + auctions: marketplaceReindexStates[0].auctionMap.size, offers: marketplaceReindexStates[0].offers.length, }); // await this.addInactiveStateItemsToDb(marketplaceReindexStates); @@ -281,12 +282,6 @@ export class MarketplacesReindexService { : [], ]); - if (auctions?.length > 0) { - marketplaceReindexState.auctions = marketplaceReindexState.auctions.concat(auctions); - } - if (orders?.length > 0) { - marketplaceReindexState.orders = marketplaceReindexState.orders.concat(orders); - } if (offers?.length > 0) { marketplaceReindexState.offers = marketplaceReindexState.offers.concat(offers); } @@ -309,9 +304,9 @@ export class MarketplacesReindexService { ) { const auctionIndex = marketplaceReindexState.marketplace.key !== ELRONDNFTSWAP_KEY - ? marketplaceReindexState.getAuctionIndexByAuctionId(eventsSetSummaries[i].auctionId) - : marketplaceReindexState.getAuctionIndexByIdentifier(eventsSetSummaries[i].identifier); - if (auctionIndex === -1) { + ? marketplaceReindexState.auctionMap.get(eventsSetSummaries[i].auctionId) + : marketplaceReindexState.auctionMap.get(eventsSetSummaries[i].identifier); + if (!auctionIndex) { marketplaceReindexState.marketplace.key !== ELRONDNFTSWAP_KEY ? missingAuctionIds.push(eventsSetSummaries[i].auctionId) : missingStateForIdentifiers.push(eventsSetSummaries[i].identifier); @@ -395,13 +390,13 @@ export class MarketplacesReindexService { private async getPaymentTokenAndNonce(marketplaceReindexState: MarketplaceReindexState, input: any): Promise<[Token, number]> { try { - const auctionIndex = + const auction = marketplaceReindexState.marketplace.key !== ELRONDNFTSWAP_KEY - ? marketplaceReindexState.getAuctionIndexByAuctionId(input.auctionId) - : marketplaceReindexState.getAuctionIndexByIdentifier(input.identifier); - const paymentNonceValue = input.paymentNonce ?? marketplaceReindexState.auctions[auctionIndex]?.paymentNonce; + ? marketplaceReindexState.auctionMap.get(input.auctionId) + : marketplaceReindexState.auctionMap.get(input.auctionId); + const paymentNonceValue = input.paymentNonce ?? auction?.paymentNonce; const paymentNonce = !Number.isNaN(paymentNonceValue) ? paymentNonceValue : 0; - const paymentTokenIdentifier = input.paymentToken ?? marketplaceReindexState.auctions[auctionIndex]?.paymentToken; + const paymentTokenIdentifier = input.paymentToken ?? auction?.paymentToken; if (paymentTokenIdentifier === mxConfig.egld) { return [ new Token({ @@ -442,15 +437,12 @@ export class MarketplacesReindexService { for (const marketplaceReindexState of marketplaceReindexStates) { // marketplaceReindexState.setStateItemsToExpiredIfOlderThanTimestamp(DateUtils.getCurrentTimestamp()); - let [inactiveAuctions, inactiveOrders, inactiveOffers] = isFinalBatch - ? marketplaceReindexState.popAllItems() - : marketplaceReindexState.popInactiveItems(); + let [inactiveAuctions, inactiveOffers] = marketplaceReindexState.popAllItems(); console.log({ - inactiveOrders: inactiveOrders.length, inactiveAuctions: inactiveAuctions.length, inactiveordersONAc: inactiveAuctions.sum((au) => au.orders?.length ?? 0), - inactiveOffers: inactiveOffers.length, + inactiveOffers: inactiveOffers?.length, }); if (inactiveAuctions.length === 0 && inactiveOffers.length === 0) { @@ -461,44 +453,29 @@ export class MarketplacesReindexService { // await this.populateAuctionMissingAssetTags(inactiveAuctions); - for (let i = 0; i < inactiveOrders.length; i++) { - if (inactiveOrders[i].auctionId < 0) { - const auctionIndex = inactiveAuctions.findIndex((a) => a.id === inactiveOrders[i].auctionId); - if (auctionIndex !== -1) { - inactiveOrders[i].auctionId = -inactiveAuctions[auctionIndex].marketplaceAuctionId; - } else { - this.logger.warn(`Corresponding auction not found`); - } - } - - if (inactiveOrders[i].id < 0) { - delete inactiveOrders[i].id; - } - } - - for (let i = 0; i < inactiveAuctions.length; i++) { - if (inactiveAuctions[i].id < 0) { - delete inactiveAuctions[i].id; - } - } + // for (let i = 0; i < inactiveAuctions.length; i++) { + // if (inactiveAuctions[i].id < 0) { + // delete inactiveAuctions[i].id; + // } + // } await this.auctionSetterService.saveBulkAuctionsOrUpdateAndFillId(inactiveAuctions); - // let tags: TagEntity[] = []; - // inactiveAuctions.map((auction) => { - // const assetTags = auction.tags.split(','); - // assetTags.map((assetTag) => { - // if (assetTag !== '') { - // tags.push( - // new TagEntity({ - // auctionId: auction.id, - // tag: assetTag.trim().slice(0, constants.dbMaxTagLength), - // auction: auction, - // }), - // ); - // } - // }); - // }); + let tags: TagEntity[] = []; + inactiveAuctions.map((auction) => { + const assetTags = auction.tags.split(','); + assetTags.map((assetTag) => { + if (assetTag !== '') { + tags.push( + new TagEntity({ + auctionId: auction.id, + tag: assetTag.trim().slice(0, constants.dbMaxTagLength), + auction: auction, + }), + ); + } + }); + }); // const saveTagsPromise = this.persistenceService.saveTagsOrIgnore(tags); @@ -523,7 +500,7 @@ export class MarketplacesReindexService { delete o.id; } }); - console.log({ orders: inactiveOrders.length, auctions: inactiveAuctions.length, offers: inactiveOffers.length }); + console.log({ auctions: inactiveAuctions.length, offers: inactiveOffers.length }); await Promise.all([ // saveTagsPromise, diff --git a/src/modules/marketplaces/models/MarketplaceReindexState.ts b/src/modules/marketplaces/models/MarketplaceReindexState.ts index 7f801c76a..9313a0d79 100644 --- a/src/modules/marketplaces/models/MarketplaceReindexState.ts +++ b/src/modules/marketplaces/models/MarketplaceReindexState.ts @@ -17,8 +17,7 @@ export class MarketplaceReindexState { marketplace: Marketplace; isFullStateInMemory: boolean; listedCollections: string[] = []; - auctions: AuctionEntity[] = []; - orders: OrderEntity[] = []; + auctionMap = new Map(); offers: OfferEntity[] = []; private auctionsTemporaryIdCounter = -1; @@ -45,52 +44,36 @@ export class MarketplaceReindexState { return this.listedCollections.includes(collection); } - getAuctionIndexByAuctionId(auctionId: number): number { - for (let i = 0; i < this.auctions.length; i++) { - if (this.auctions[i].marketplaceAuctionId === auctionId) { - return i; - } - } - return -1; - } - - getAuctionIndexByIdentifier(identifier: string): number { - return this.auctions.findIndex((a) => a.identifier === identifier && a.status === AuctionStatusEnum.Running); - } - getOfferIndexByOfferId(offerId: number): number { return this.offers.findIndex((o) => o.marketplaceOfferId === offerId); } - createOrder(auctionIndex: number, input: any, status: OrderStatusEnum, paymentToken: Token, paymentNonce?: number): OrderEntity { + createOrder(auction: AuctionEntity, input: any, status: OrderStatusEnum, paymentToken: Token, paymentNonce?: number): OrderEntity { const modifiedDate = DateUtils.getUtcDateFromTimestamp(input.timestamp); const price = input.price ?? input.currentBid; return new OrderEntity({ id: this.getNewOrderId(), creationDate: modifiedDate, modifiedDate, - auctionId: this.auctions[auctionIndex].id, ownerAddress: input.address, priceToken: paymentToken.identifier, priceNonce: paymentNonce ?? 0, - priceAmount: new BigNumber(price !== '0' ? price : this.auctions[auctionIndex].maxBid).toFixed(), - priceAmountDenominated: - price !== '0' ? BigNumberUtils.denominateAmount(price, paymentToken.decimals) : this.auctions[auctionIndex].maxBidDenominated, + priceAmount: new BigNumber(price !== '0' ? price : auction.maxBid).toFixed(), + priceAmountDenominated: price !== '0' ? BigNumberUtils.denominateAmount(price, paymentToken.decimals) : auction.maxBidDenominated, blockHash: input.blockHash ?? '', marketplaceKey: this.marketplace.key, - boughtTokensNo: this.auctions[auctionIndex].type === AuctionTypeEnum.Nft ? null : input.itemsCount, + boughtTokensNo: auction.type === AuctionTypeEnum.Nft ? null : input.itemsCount, status: status, }); } - setAuctionOrderWinnerStatusAndReturnId(auctionId: number, status: OrderStatusEnum, modifiedDate?: Date): number { - const selectedAuction = this.auctions[auctionId]; - if (!selectedAuction) { + setAuctionOrderWinnerStatusAndReturnId(auction: AuctionEntity, status: OrderStatusEnum, modifiedDate?: Date): number { + if (!auction) { return -1; } - const activeOrders = selectedAuction.orders.filter((o) => o.status === OrderStatusEnum.Active); - if (activeOrders.length === 0) { + const activeOrders = auction.orders?.filter((o) => o.status === OrderStatusEnum.Active); + if (!activeOrders?.length) { return -1; } @@ -107,10 +90,7 @@ export class MarketplaceReindexState { return winnerOrder.id; } - - setInactiveOrdersForAuction(auctionId: number, modifiedDate: Date, exceptWinnerId?: number): void { - const auction = this.auctions[auctionId]; - + setInactiveOrdersForAuction(auction: AuctionEntity, modifiedDate: Date, exceptWinnerId?: number): void { if (auction && auction.orders) { for (const order of auction.orders) { if (order.status === OrderStatusEnum.Active && order.id !== exceptWinnerId) { @@ -122,101 +102,31 @@ export class MarketplaceReindexState { } setStateItemsToExpiredIfOlderThanTimestamp(timestamp: number): void { - this.setAuctionsAndOrdersToExpiredIfOlderThanTimestamp(timestamp); this.setOffersToExpiredIfOlderThanTimestamp(timestamp); } - popAllItems(): [AuctionEntity[], OrderEntity[], OfferEntity[]] { - const inactiveAuctions = this.auctions; - const inactiveOrders = this.orders; + popAllItems(): [AuctionEntity[], OfferEntity[]] { + const inactiveAuctions = [...this.auctionMap.values()]; const inactiveOffers = this.offers; - this.auctions = []; - this.orders = []; this.offers = []; - return [inactiveAuctions, inactiveOrders, inactiveOffers]; - } - - popInactiveItems(): [AuctionEntity[], OrderEntity[], OfferEntity[]] { - const inactiveAuctionStatuses = [AuctionStatusEnum.Closed, AuctionStatusEnum.Ended]; - const inactiveOfferStatuses = [OfferStatusEnum.Accepted, OfferStatusEnum.Closed, OfferStatusEnum.Expired]; - - let inactiveAuctions = []; - let inactiveOrders = []; - let inactiveOffers = []; - - if (this.marketplace.key === 'elrondapes') { - console.log( - `orders state`, - this.orders.map((o) => o.id), - this.orders.map((o) => o.auctionId), - ); - } - - for (let i = 0; i < this.auctions.length; i++) { - const isInactiveAuction = inactiveAuctionStatuses.includes(this.auctions[i].status); - const isOldAuction = - this.auctions.length > constants.marketplaceReindexDataMaxInMemoryItems && - i < this.auctions.length - constants.marketplaceReindexDataMaxInMemoryItems; - - if (isInactiveAuction || isOldAuction) { - inactiveAuctions.push(this.auctions.splice(i--, 1)[0]); - } - } - - for (let i = 0; i < this.orders.length; i++) { - const isInactiveOrOldOrder = inactiveAuctions.findIndex((a) => a.id === this.orders[i].auctionId) !== -1; - - if (isInactiveOrOldOrder) { - inactiveOrders.push(this.orders.splice(i--, 1)[0]); - } - } - - for (let i = 0; i < this.offers.length; i++) { - const isInactiveOffer = inactiveOfferStatuses.includes(this.offers[i].status); - const isOldOffer = - this.offers.length > constants.marketplaceReindexDataMaxInMemoryItems && - i < this.offers.length - constants.marketplaceReindexDataMaxInMemoryItems; - - if (isInactiveOffer || isOldOffer) { - inactiveOffers.push(this.offers.splice(i--, 1)[0]); - } - } - - return [inactiveAuctions, inactiveOrders, inactiveOffers]; - } - - public updateOrderListForAuction(auctionIndex: number, order: OrderEntity) { - const existingOrders = this.auctions[auctionIndex].orders || []; - - this.auctions[auctionIndex].orders = [ - ...existingOrders.map((existingOrder) => ({ ...existingOrder, status: OrderStatusEnum.Inactive })), - order, - ]; + return [inactiveAuctions, inactiveOffers]; } - public updateAuctionStatus(auctionIndex: number, blockHash: string, status: AuctionStatusEnum, timestamp: number): void { - const currentAuction = this.auctions[auctionIndex]; - if (currentAuction) { - const updatedAuction = { - ...currentAuction, - status, - blockHash: currentAuction.blockHash ?? blockHash, - modifiedDate: DateUtils.getUtcDateFromTimestamp(timestamp), - }; + public updateOrderListForAuction(auction: AuctionEntity, order: OrderEntity) { + const existingOrders = auction.orders || []; - this.auctions[auctionIndex] = updatedAuction; - } + auction.orders = [...existingOrders.map((existingOrder) => ({ ...existingOrder, status: OrderStatusEnum.Inactive })), order]; } - deleteAuctionIfDuplicates(marketplaceAuctionId: number) { - if (this.isFullStateInMemory) { - return; - } + public updateAuctionStatus(auction: AuctionEntity, blockHash: string, status: AuctionStatusEnum, timestamp: number): void { + const updatedAuction = { + ...auction, + status, + blockHash: auction.blockHash ?? blockHash, + modifiedDate: DateUtils.getUtcDateFromTimestamp(timestamp), + }; - let index; - while ((index = this.auctions.findIndex((a) => a.marketplaceAuctionId === marketplaceAuctionId)) !== -1) { - this.auctions.splice(index, 1); - } + this.auctionMap[auction.marketplaceAuctionId] = updatedAuction; } deleteOfferIfDuplicates(marketplaceOfferId: number) { @@ -231,21 +141,6 @@ export class MarketplaceReindexState { } while (index !== -1); } - private setAuctionsAndOrdersToExpiredIfOlderThanTimestamp(timestamp: number): void { - const runningAuctions = this.auctions?.filter((a) => a.status === AuctionStatusEnum.Running); - for (let i = 0; i < runningAuctions.length; i++) { - if (runningAuctions[i].endDate > 0 && runningAuctions[i].endDate < timestamp) { - runningAuctions[i].status = AuctionStatusEnum.Claimable; - const winnerOrderId = this.setAuctionOrderWinnerStatusAndReturnId(runningAuctions[i].id, OrderStatusEnum.Active); - this.setInactiveOrdersForAuction( - runningAuctions[i].id, - DateUtils.getUtcDateFromTimestamp(runningAuctions[i].endDate), - winnerOrderId, - ); - } - } - } - private setOffersToExpiredIfOlderThanTimestamp(timestamp: number): void { for (let i = 0; i < this.offers.length; i++) { if (this.offers[i].status === OfferStatusEnum.Active && this.offers[i].endDate && this.offers[i].endDate < timestamp) { From 2101e3b73bd2ccec2aa60d14c1a4cc72e68aa7d7 Mon Sep 17 00:00:00 2001 From: danielailie Date: Thu, 16 Nov 2023 15:52:52 +0200 Subject: [PATCH 12/18] Update auction repo --- src/db/auctions/auctions.repository.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/db/auctions/auctions.repository.ts b/src/db/auctions/auctions.repository.ts index 536d422c8..876d38244 100644 --- a/src/db/auctions/auctions.repository.ts +++ b/src/db/auctions/auctions.repository.ts @@ -444,10 +444,8 @@ export class AuctionsRepository { const connection = this.auctionsRepository.manager.connection; await connection.transaction(async (transactionalEntityManager) => { - const queryBuilder = transactionalEntityManager.createQueryBuilder().from(AuctionEntity, 'auction'); - for (let i = 0; i < auctions.length; i += batchSize) { - const currentQueryBuilder = queryBuilder.clone(); + const currentQueryBuilder = transactionalEntityManager.createQueryBuilder().from(AuctionEntity, 'auction'); const batch = auctions.slice(i, i + batchSize); console.log('Processing batch number', i); @@ -500,9 +498,10 @@ export class AuctionsRepository { `); } } + + await currentQueryBuilder.execute(); } - await currentQueryBuilder.execute(); cpu.stop(`batch ${i}`); } From 93adbc54eada218c6ecf221e0f08084f342d00d1 Mon Sep 17 00:00:00 2001 From: danielailie Date: Mon, 20 Nov 2023 14:10:41 +0200 Subject: [PATCH 13/18] Update full history reindex --- src/db/auctions/auctions.repository.ts | 144 ++++++++++++------ .../reindex-auction-bought.handler.ts | 6 +- .../reindex-auction-started.handler.ts | 1 - .../models/MarketplaceReindexState.ts | 1 - 4 files changed, 99 insertions(+), 53 deletions(-) diff --git a/src/db/auctions/auctions.repository.ts b/src/db/auctions/auctions.repository.ts index 876d38244..cb8b37783 100644 --- a/src/db/auctions/auctions.repository.ts +++ b/src/db/auctions/auctions.repository.ts @@ -34,6 +34,7 @@ import { } from './sql.queries'; import { CpuProfiler } from '@multiversx/sdk-nestjs-monitoring'; import { OrderEntity } from '../orders'; +import { TagEntity } from './tags.entity'; @Injectable() export class AuctionsRepository { @@ -435,6 +436,67 @@ export class AuctionsRepository { .execute(); } + // async saveBulkAuctions(auctions: AuctionEntity[]): Promise { + // await this.auctionsRepository.save(auctions, { + // chunk: 1000, + // }); + // } + + // async saveBulkAuctionsOrUpdateAndFillId(auctions: AuctionEntity[]): Promise { + // const batchSize = 1000; + // if (auctions.length === 0) { + // return; + // } + + // const connection = this.auctionsRepository.manager.connection; + + // await connection.transaction(async (transactionalEntityManager) => { + // for (let i = 0; i < auctions.length; i += batchSize) { + // const batch = auctions.slice(i, i + batchSize); + // console.log('Processing batch number', i); + + // const cpu = new CpuProfiler(); + // // for (const item of batch) { + // const currentQueryBuilder = transactionalEntityManager + // .createQueryBuilder() + // .from(AuctionEntity, 'auction') + // .insert() + // .values(batch) + // .orUpdate({ + // conflict_target: ['marketplaceKey', 'marketplaceAuctionId'], + // overwrite: [ + // 'collection', + // 'nrAuctionedTokens', + // 'identifier', + // 'nonce', + // 'status', + // 'type', + // 'paymentToken', + // 'paymentNonce', + // 'ownerAddress', + // 'minBidDiff', + // 'minBid', + // 'minBidDenominated', + // 'maxBid', + // 'maxBidDenominated', + // 'startDate', + // 'tags', + // 'blockHash', + // ], + // }); + // console.log(currentQueryBuilder.getQueryAndParameters()); + // // Include related orders + + // await currentQueryBuilder.execute(); + // // } + + // cpu.stop(`batch ${i}`); + // } + + // console.log('Bulk insert or update completed successfully.'); + // }); + // } + async saveBulkAuctionsOrUpdateAndFillId(auctions: AuctionEntity[]): Promise { const batchSize = 1000; if (auctions.length === 0) { @@ -445,61 +507,47 @@ export class AuctionsRepository { await connection.transaction(async (transactionalEntityManager) => { for (let i = 0; i < auctions.length; i += batchSize) { - const currentQueryBuilder = transactionalEntityManager.createQueryBuilder().from(AuctionEntity, 'auction'); const batch = auctions.slice(i, i + batchSize); console.log('Processing batch number', i); const cpu = new CpuProfiler(); + const response = await transactionalEntityManager + .createQueryBuilder() + .from(AuctionEntity, 'auction') + .insert() + .into(AuctionEntity) + .values(batch) + .execute(); + console.log({ auctions: response?.identifiers?.length }); + + let orders = []; + let tags = []; for (const item of batch) { - currentQueryBuilder.insert().values(item).onConflict(`("marketplaceKey", "marketplaceAuctionId") DO UPDATE SET - "collection" = EXCLUDED."collection", - "nrAuctionedTokens" = EXCLUDED."nrAuctionedTokens", - "identifier" = EXCLUDED."identifier", - "nonce" = EXCLUDED."nonce", - "status" = EXCLUDED."status", - "type" = EXCLUDED."type", - "paymentToken" = EXCLUDED."paymentToken", - "paymentNonce" = EXCLUDED."paymentNonce", - "ownerAddress" = EXCLUDED."ownerAddress", - "minBidDiff" = EXCLUDED."minBidDiff", - "minBid" = EXCLUDED."minBid", - "minBidDenominated" = EXCLUDED."minBidDenominated", - "maxBid" = EXCLUDED."maxBid", - "maxBidDenominated" = EXCLUDED."maxBidDenominated", - "startDate" = EXCLUDED."startDate", - "endDate" = EXCLUDED."endDate", - "tags" = EXCLUDED."tags", - "blockHash" = EXCLUDED."blockHash" - `); - - // Include related orders - if (item.orders && item.orders.length > 0) { - for (const order of item.orders) { - currentQueryBuilder.insert().into(OrderEntity).values({ - priceToken: order.priceToken, - priceAmount: order.priceAmount, - priceAmountDenominated: order.priceAmountDenominated, - priceNonce: order.priceNonce, - status: order.status, - ownerAddress: order.ownerAddress, - boughtTokensNo: order.boughtTokensNo, - auctionId: order.auctionId, - blockHash: order.blockHash, - marketplaceKey: order.marketplaceKey, - }).onConflict(`("auctionId", "marketplaceKey") DO UPDATE SET - "priceToken" = EXCLUDED."priceToken", - "priceAmount" = EXCLUDED."priceAmount", - "priceAmountDenominated" = EXCLUDED."priceAmountDenominated", - "priceNonce" = EXCLUDED."priceNonce", - "status" = EXCLUDED."status", - "ownerAddress" = EXCLUDED."ownerAddress", - "boughtTokensNo" = EXCLUDED."boughtTokensNo", - "blockHash" = EXCLUDED."blockHash" - `); - } + if (item.orders) { + item.orders.forEach((i) => { + i.auction = item; + }); + orders.push(...item.orders); + } + if (item.tagEntities && item.tagEntities.length > 0) { + console.log(item.tagEntities); + item.tagEntities.forEach((i) => { + i.auction = item; + }); + + tags.push(...item.tagEntities); } + } + if (orders.length) { + const res = await transactionalEntityManager.createQueryBuilder().insert().into(OrderEntity).values(orders).execute(); + + console.log({ orders: res?.identifiers?.length }); + } + + if (tags.length) { + const res = await transactionalEntityManager.createQueryBuilder().insert().into(TagEntity).values(tags).execute(); - await currentQueryBuilder.execute(); + console.log({ orders: res?.identifiers?.length }); } cpu.stop(`batch ${i}`); diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bought.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bought.handler.ts index f4848fd4c..324e98060 100644 --- a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bought.handler.ts +++ b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bought.handler.ts @@ -24,7 +24,7 @@ export class ReindexAuctionBoughtHandler { const order = marketplaceReindexState.createOrder(auction, input, OrderStatusEnum.Bought, paymentToken, paymentNonce); if (auction.nrAuctionedTokens > 1) { - const totalBought = this.getTotalBoughtTokensForAuction(auction.id, auction.orders); + const totalBought = this.getTotalBoughtTokensForAuction(auction.orders); if (auction.nrAuctionedTokens === totalBought) { marketplaceReindexState.updateAuctionStatus(auction, input.blockHash, AuctionStatusEnum.Ended, input.timestamp); @@ -34,11 +34,11 @@ export class ReindexAuctionBoughtHandler { marketplaceReindexState.updateOrderListForAuction(auction, order); } - private getTotalBoughtTokensForAuction(auctionId: number, orders: OrderEntity[]): number { + private getTotalBoughtTokensForAuction(orders: OrderEntity[]): number { let totalBought = 0; if (orders?.length) { orders - .filter((o) => o.auctionId === auctionId && o.status === OrderStatusEnum.Bought) + .filter((o) => o.status === OrderStatusEnum.Bought) .forEach((o) => { totalBought += parseInt(o.boughtTokensNo) ? parseInt(o.boughtTokensNo) : 1; }); diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-started.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-started.handler.ts index d832da007..3050a0e22 100644 --- a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-started.handler.ts +++ b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-started.handler.ts @@ -26,7 +26,6 @@ export class ReindexAuctionStartedHandler { const auction = new AuctionEntity({ creationDate: modifiedDate, modifiedDate, - id: marketplaceReindexState.getNewAuctionId(), marketplaceAuctionId: input.auctionId !== 0 ? input.auctionId : marketplaceReindexState.auctionMap.size + 1, identifier: input.identifier, collection: input.collection, diff --git a/src/modules/marketplaces/models/MarketplaceReindexState.ts b/src/modules/marketplaces/models/MarketplaceReindexState.ts index 9313a0d79..4e109d4b9 100644 --- a/src/modules/marketplaces/models/MarketplaceReindexState.ts +++ b/src/modules/marketplaces/models/MarketplaceReindexState.ts @@ -52,7 +52,6 @@ export class MarketplaceReindexState { const modifiedDate = DateUtils.getUtcDateFromTimestamp(input.timestamp); const price = input.price ?? input.currentBid; return new OrderEntity({ - id: this.getNewOrderId(), creationDate: modifiedDate, modifiedDate, ownerAddress: input.address, From aee05d16a810a2bf0ef081928176f5f2282306a5 Mon Sep 17 00:00:00 2001 From: danielailie Date: Mon, 20 Nov 2023 15:51:47 +0200 Subject: [PATCH 14/18] Fix auctions status --- .../marketplaces/models/MarketplaceReindexState.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/modules/marketplaces/models/MarketplaceReindexState.ts b/src/modules/marketplaces/models/MarketplaceReindexState.ts index 4e109d4b9..4628a308a 100644 --- a/src/modules/marketplaces/models/MarketplaceReindexState.ts +++ b/src/modules/marketplaces/models/MarketplaceReindexState.ts @@ -118,14 +118,9 @@ export class MarketplaceReindexState { } public updateAuctionStatus(auction: AuctionEntity, blockHash: string, status: AuctionStatusEnum, timestamp: number): void { - const updatedAuction = { - ...auction, - status, - blockHash: auction.blockHash ?? blockHash, - modifiedDate: DateUtils.getUtcDateFromTimestamp(timestamp), - }; - - this.auctionMap[auction.marketplaceAuctionId] = updatedAuction; + auction.status = status; + auction.blockHash = auction.blockHash ?? blockHash; + auction.modifiedDate = DateUtils.getUtcDateFromTimestamp(timestamp); } deleteOfferIfDuplicates(marketplaceOfferId: number) { From 91021eab2fc556909931574c4e53ba7500f23162 Mon Sep 17 00:00:00 2001 From: danielailie Date: Tue, 21 Nov 2023 11:40:07 +0200 Subject: [PATCH 15/18] Update Status to claimable --- src/modules/marketplaces/models/MarketplaceReindexState.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/modules/marketplaces/models/MarketplaceReindexState.ts b/src/modules/marketplaces/models/MarketplaceReindexState.ts index 4628a308a..1fd43779e 100644 --- a/src/modules/marketplaces/models/MarketplaceReindexState.ts +++ b/src/modules/marketplaces/models/MarketplaceReindexState.ts @@ -118,6 +118,9 @@ export class MarketplaceReindexState { } public updateAuctionStatus(auction: AuctionEntity, blockHash: string, status: AuctionStatusEnum, timestamp: number): void { + if (auction.endDate > 0 && auction.endDate <= DateUtils.getCurrentTimestamp()) { + status = AuctionStatusEnum.Claimable; + } auction.status = status; auction.blockHash = auction.blockHash ?? blockHash; auction.modifiedDate = DateUtils.getUtcDateFromTimestamp(timestamp); From f93c4e7ea64c7d493e01c034739caba757ab21bd Mon Sep 17 00:00:00 2001 From: danielailie Date: Tue, 21 Nov 2023 11:44:35 +0200 Subject: [PATCH 16/18] Update status if ended auction --- .../reindex-auction-started.handler.ts | 6 +++++- src/modules/marketplaces/models/MarketplaceReindexState.ts | 3 --- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-started.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-started.handler.ts index 3050a0e22..a30144a04 100644 --- a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-started.handler.ts +++ b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-started.handler.ts @@ -17,11 +17,15 @@ export class ReindexAuctionStartedHandler { handle(input: AuctionStartedSummary, marketplaceReindexState: MarketplaceReindexState, paymentToken: Token, paymentNonce: number): void { const nonce = BinaryUtils.hexToNumber(input.nonce); const itemsCount = parseInt(input.itemsCount); + let status = AuctionStatusEnum.Running; const modifiedDate = DateUtils.getUtcDateFromTimestamp(input.timestamp); const startTime = Number.isNaN(input.startTime) ? input.timestamp : input.startTime; const endTime = input.endTime > 0 ? input.endTime : 0; const minBidDenominated = BigNumberUtils.denominateAmount(input.minBid, paymentToken.decimals); const maxBidDenominated = BigNumberUtils.denominateAmount(input.maxBid !== 'NaN' ? input.maxBid : '0', paymentToken.decimals); + if (endTime > 0 && endTime <= DateUtils.getCurrentTimestamp()) { + status = AuctionStatusEnum.Claimable; + } const auction = new AuctionEntity({ creationDate: modifiedDate, @@ -31,7 +35,7 @@ export class ReindexAuctionStartedHandler { collection: input.collection, nonce: nonce, nrAuctionedTokens: itemsCount, - status: AuctionStatusEnum.Running, + status: status, type: input.auctionType, paymentToken: paymentToken.identifier, paymentNonce, diff --git a/src/modules/marketplaces/models/MarketplaceReindexState.ts b/src/modules/marketplaces/models/MarketplaceReindexState.ts index 1fd43779e..4628a308a 100644 --- a/src/modules/marketplaces/models/MarketplaceReindexState.ts +++ b/src/modules/marketplaces/models/MarketplaceReindexState.ts @@ -118,9 +118,6 @@ export class MarketplaceReindexState { } public updateAuctionStatus(auction: AuctionEntity, blockHash: string, status: AuctionStatusEnum, timestamp: number): void { - if (auction.endDate > 0 && auction.endDate <= DateUtils.getCurrentTimestamp()) { - status = AuctionStatusEnum.Claimable; - } auction.status = status; auction.blockHash = auction.blockHash ?? blockHash; auction.modifiedDate = DateUtils.getUtcDateFromTimestamp(timestamp); From 7b9b8608a2c64229741639cfc4b0086576d84953 Mon Sep 17 00:00:00 2001 From: danielailie Date: Tue, 21 Nov 2023 12:14:39 +0200 Subject: [PATCH 17/18] add extra event --- .../marketplaces-reindex-events-summary.service.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/modules/marketplaces/marketplaces-reindex-events-summary.service.ts b/src/modules/marketplaces/marketplaces-reindex-events-summary.service.ts index f6dc28eff..ebfc5dbce 100644 --- a/src/modules/marketplaces/marketplaces-reindex-events-summary.service.ts +++ b/src/modules/marketplaces/marketplaces-reindex-events-summary.service.ts @@ -105,7 +105,8 @@ export class MarketplacesReindexEventsSummaryService { case AuctionEventEnum.WithdrawOffer: { return OfferClosedSummary.fromWithdrawOfferEventAndTx(event, txData); } - case AuctionEventEnum.WithdrawAuctionAndAcceptOffer: { + case AuctionEventEnum.WithdrawAuctionAndAcceptOffer: + case ExternalAuctionEventEnum.AcceptOfferFromAuction: { if (event.hasEventTopicIdentifier(AuctionEventEnum.Accept_offer_token_event)) { return OfferAcceptedSummary.fromAcceptOfferEventAndTx(event, txData, marketplace); } else { From c0eb5eae0b42441e0e292a3d54a3e41da5fbfcf0 Mon Sep 17 00:00:00 2001 From: danielailie Date: Tue, 21 Nov 2023 14:03:59 +0200 Subject: [PATCH 18/18] Fix auction buy processing --- .../reindex-auction-bought.handler.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bought.handler.ts b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bought.handler.ts index 324e98060..163ca0b25 100644 --- a/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bought.handler.ts +++ b/src/modules/marketplaces/marketplaces-reindex-handlers/reindex-auction-bought.handler.ts @@ -29,8 +29,9 @@ export class ReindexAuctionBoughtHandler { if (auction.nrAuctionedTokens === totalBought) { marketplaceReindexState.updateAuctionStatus(auction, input.blockHash, AuctionStatusEnum.Ended, input.timestamp); } + } else { + marketplaceReindexState.updateAuctionStatus(auction, input.blockHash, AuctionStatusEnum.Ended, input.timestamp); } - marketplaceReindexState.updateOrderListForAuction(auction, order); }