diff --git a/src/client/java/teammates/client/scripts/sql/PatchCreatedAtAccountRequest.java b/src/client/java/teammates/client/scripts/sql/PatchCreatedAtAccountRequest.java new file mode 100644 index 00000000000..e23b330ec1c --- /dev/null +++ b/src/client/java/teammates/client/scripts/sql/PatchCreatedAtAccountRequest.java @@ -0,0 +1,246 @@ +package teammates.client.scripts.sql; + +// CHECKSTYLE.OFF:ImportOrder +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicLong; + +import com.google.cloud.datastore.Cursor; +import com.google.cloud.datastore.QueryResults; +import com.googlecode.objectify.cmd.Query; + +import jakarta.persistence.criteria.CriteriaBuilder; +import jakarta.persistence.criteria.CriteriaQuery; +import jakarta.persistence.criteria.Root; + +import teammates.client.connector.DatastoreClient; +import teammates.client.util.ClientProperties; +import teammates.common.util.Const; +import teammates.common.util.HibernateUtil; +import teammates.test.FileHelper; + +// CHECKSTYLE.ON:ImportOrder +/** + * Patch createdAt attribute for Account Request + * Assumes that the notification was previously migrated using DataMigrationForAccountRequestSql.java + */ +@SuppressWarnings("PMD") +public class PatchCreatedAtAccountRequest extends DatastoreClient { + // the folder where the cursor position and console output is saved as a file + private static final String BASE_LOG_URI = "src/client/java/teammates/client/scripts/log/"; + + // 100 is the optimal batch size as there won't be too much time interval + // between read and save (if transaction is not used) + // cannot set number greater than 300 + // see + // https://stackoverflow.com/questions/41499505/objectify-queries-setting-limit-above-300-does-not-work + private static final int BATCH_SIZE = 100; + + // Creates the folder that will contain the stored log. + static { + new File(BASE_LOG_URI).mkdir(); + } + + AtomicLong numberOfScannedKey; + AtomicLong numberOfAffectedEntities; + AtomicLong numberOfUpdatedEntities; + + private PatchCreatedAtAccountRequest() { + numberOfScannedKey = new AtomicLong(); + numberOfAffectedEntities = new AtomicLong(); + numberOfUpdatedEntities = new AtomicLong(); + + String connectionUrl = ClientProperties.SCRIPT_API_URL; + String username = ClientProperties.SCRIPT_API_NAME; + String password = ClientProperties.SCRIPT_API_PASSWORD; + + HibernateUtil.buildSessionFactory(connectionUrl, username, password); + } + + public static void main(String[] args) { + new PatchCreatedAtAccountRequest().doOperationRemotely(); + } + + /** + * Returns the log prefix. + */ + protected String getLogPrefix() { + return String.format("Account Request Patch Migration:"); + } + + private boolean isPreview() { + return false; + } + + /** + * Returns whether the account has been migrated. + */ + protected boolean isMigrationNeeded(teammates.storage.entity.AccountRequest entity) { + return true; + } + + /** + * Returns the filter query. + */ + protected Query getFilterQuery() { + return ofy().load().type(teammates.storage.entity.AccountRequest.class); + } + + private void doMigration(teammates.storage.entity.AccountRequest entity) { + try { + if (!isMigrationNeeded(entity)) { + return; + } + if (!isPreview()) { + migrateEntity(entity); + } + } catch (Exception e) { + logError("Problem patching account request " + entity); + logError(e.getMessage()); + } + } + + /** + * Migrates the entity. In this case, add entity to buffer. + */ + protected void migrateEntity(teammates.storage.entity.AccountRequest oldEntity) { + HibernateUtil.beginTransaction(); + + CriteriaBuilder cb = HibernateUtil.getCriteriaBuilder(); + CriteriaQuery cr = cb.createQuery(teammates.storage.sqlentity.AccountRequest.class); + Root root = cr.from(teammates.storage.sqlentity.AccountRequest.class); + + cr.select(root).where(cb.equal(root.get("institute"), oldEntity.getInstitute())) + .where(cb.equal(root.get("email"), oldEntity.getEmail())) + .where(cb.equal(root.get("name"), oldEntity.getName())); + + List matchingAccounts = HibernateUtil.createQuery(cr).getResultList(); + + if (matchingAccounts.size() > 1) { + throw new Error("More than one matching account request found"); + } else if (matchingAccounts.size() == 0){ + throw new Error("No matching account found"); + } + + // Get first items since there is guaranteed to be one + teammates.storage.sqlentity.AccountRequest newAccountReq = matchingAccounts.get(0); + newAccountReq.setCreatedAt(oldEntity.getCreatedAt()); + HibernateUtil.commitTransaction(); + numberOfAffectedEntities.incrementAndGet(); + numberOfUpdatedEntities.incrementAndGet(); + } + + @Override + protected void doOperation() { + log("Running " + getClass().getSimpleName() + "..."); + log("Preview: " + isPreview()); + + Cursor cursor = readPositionOfCursorFromFile().orElse(null); + if (cursor == null) { + log("Start from the beginning"); + } else { + log("Start from cursor position: " + cursor.toUrlSafe()); + } + + boolean shouldContinue = true; + while (shouldContinue) { + shouldContinue = false; + Query filterQueryKeys = getFilterQuery().limit(BATCH_SIZE); + if (cursor != null) { + filterQueryKeys = filterQueryKeys.startAt(cursor); + } + QueryResults iterator; + + iterator = filterQueryKeys.iterator(); + + while (iterator.hasNext()) { + shouldContinue = true; + + doMigration(iterator.next()); + + numberOfScannedKey.incrementAndGet(); + } + + if (shouldContinue) { + cursor = iterator.getCursorAfter(); + savePositionOfCursorToFile(cursor); + log(String.format("Cursor Position: %s", cursor.toUrlSafe())); + log(String.format("Number Of Entity Key Scanned: %d", numberOfScannedKey.get())); + log(String.format("Number Of Entity affected: %d", numberOfAffectedEntities.get())); + log(String.format("Number Of Entity updated: %d", numberOfUpdatedEntities.get())); + } + } + + deleteCursorPositionFile(); + log(isPreview() ? "Preview Completed!" : "Migration Completed!"); + log("Total number of entities: " + numberOfScannedKey.get()); + log("Number of affected entities: " + numberOfAffectedEntities.get()); + log("Number of updated entities: " + numberOfUpdatedEntities.get()); + } + + /** + * Saves the cursor position to a file so it can be used in the next run. + */ + private void savePositionOfCursorToFile(Cursor cursor) { + try { + FileHelper.saveFile( + BASE_LOG_URI + this.getClass().getSimpleName() + ".cursor", cursor.toUrlSafe()); + } catch (IOException e) { + logError("Fail to save cursor position " + e.getMessage()); + } + } + + /** + * Reads the cursor position from the saved file. + * + * @return cursor if the file can be properly decoded. + */ + private Optional readPositionOfCursorFromFile() { + try { + String cursorPosition = FileHelper.readFile(BASE_LOG_URI + this.getClass().getSimpleName() + ".cursor"); + return Optional.of(Cursor.fromUrlSafe(cursorPosition)); + } catch (IOException | IllegalArgumentException e) { + return Optional.empty(); + } + } + + /** + * Deletes the cursor position file. + */ + private void deleteCursorPositionFile() { + FileHelper.deleteFile(BASE_LOG_URI + this.getClass().getSimpleName() + ".cursor"); + } + + /** + * Logs a comment. + */ + protected void log(String logLine) { + System.out.println(String.format("%s %s", getLogPrefix(), logLine)); + + Path logPath = Paths.get(BASE_LOG_URI + this.getClass().getSimpleName() + ".log"); + try (OutputStream logFile = Files.newOutputStream(logPath, + StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.APPEND)) { + logFile.write((logLine + System.lineSeparator()).getBytes(Const.ENCODING)); + } catch (Exception e) { + System.err.println("Error writing log line: " + logLine); + System.err.println(e.getMessage()); + } + } + + /** + * Logs an error and persists it to the disk. + */ + protected void logError(String logLine) { + System.err.println(logLine); + + log("[ERROR]" + logLine); + } + +} diff --git a/src/client/java/teammates/client/scripts/sql/PatchCreatedAtTimeNotification.java b/src/client/java/teammates/client/scripts/sql/PatchCreatedAtTimeNotification.java new file mode 100644 index 00000000000..789a0cc6497 --- /dev/null +++ b/src/client/java/teammates/client/scripts/sql/PatchCreatedAtTimeNotification.java @@ -0,0 +1,242 @@ +package teammates.client.scripts.sql; + +// CHECKSTYLE.OFF:ImportOrder +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicLong; + +import com.google.cloud.datastore.Cursor; +import com.google.cloud.datastore.QueryResults; +import com.googlecode.objectify.cmd.Query; + +import jakarta.persistence.criteria.CriteriaBuilder; +import jakarta.persistence.criteria.CriteriaQuery; +import jakarta.persistence.criteria.Root; + +import teammates.client.connector.DatastoreClient; +import teammates.client.util.ClientProperties; +import teammates.common.util.Const; +import teammates.common.util.HibernateUtil; +import teammates.test.FileHelper; + +// CHECKSTYLE.ON:ImportOrder +/** + * Patch createdAt attribute for Notification + * Assumes that the notification was previously migrated using DataMigrationForNotificationSql.java + */ +@SuppressWarnings("PMD") +public class PatchCreatedAtTimeNotification extends DatastoreClient { + // the folder where the cursor position and console output is saved as a file + private static final String BASE_LOG_URI = "src/client/java/teammates/client/scripts/log/"; + + // 100 is the optimal batch size as there won't be too much time interval + // between read and save (if transaction is not used) + // cannot set number greater than 300 + // see + // https://stackoverflow.com/questions/41499505/objectify-queries-setting-limit-above-300-does-not-work + private static final int BATCH_SIZE = 100; + + // Creates the folder that will contain the stored log. + static { + new File(BASE_LOG_URI).mkdir(); + } + + AtomicLong numberOfScannedKey; + AtomicLong numberOfAffectedEntities; + AtomicLong numberOfUpdatedEntities; + + private PatchCreatedAtTimeNotification() { + numberOfScannedKey = new AtomicLong(); + numberOfAffectedEntities = new AtomicLong(); + numberOfUpdatedEntities = new AtomicLong(); + + String connectionUrl = ClientProperties.SCRIPT_API_URL; + String username = ClientProperties.SCRIPT_API_NAME; + String password = ClientProperties.SCRIPT_API_PASSWORD; + + HibernateUtil.buildSessionFactory(connectionUrl, username, password); + } + + public static void main(String[] args) { + new PatchCreatedAtTimeNotification().doOperationRemotely(); + } + + /** + * Returns the log prefix. + */ + protected String getLogPrefix() { + return String.format("Notification Patch Migration:"); + } + + private boolean isPreview() { + return false; + } + + /** + * Returns whether the account has been migrated. + */ + protected boolean isMigrationNeeded(teammates.storage.entity.Notification entity) { + return true; + } + + /** + * Returns the filter query. + */ + protected Query getFilterQuery() { + return ofy().load().type(teammates.storage.entity.Notification.class); + } + + private void doMigration(teammates.storage.entity.Notification entity) { + try { + if (!isMigrationNeeded(entity)) { + return; + } + if (!isPreview()) { + migrateEntity(entity); + } + } catch (Exception e) { + logError("Problem patching usage stats " + entity); + logError(e.getMessage()); + } + } + + /** + * Migrates the entity. In this case, add entity to buffer. + */ + protected void migrateEntity(teammates.storage.entity.Notification oldEntity) { + HibernateUtil.beginTransaction(); + + CriteriaBuilder cb = HibernateUtil.getCriteriaBuilder(); + CriteriaQuery cr = cb.createQuery(teammates.storage.sqlentity.Notification.class); + Root root = cr.from(teammates.storage.sqlentity.Notification.class); + + cr.select(root).where(cb.equal(root.get("title"), oldEntity.getTitle())); + List matchingNotifs = HibernateUtil.createQuery(cr).getResultList(); + if (matchingNotifs.size() > 1) { + throw new Error("More than one matching notification found with title"); + } else if (matchingNotifs.size() == 0){ + throw new Error("No notification found with title"); + } + + // Get first items since there is guaranteed to be one + teammates.storage.sqlentity.Notification newNotif = matchingNotifs.get(0); + newNotif.setCreatedAt(oldEntity.getCreatedAt()); + HibernateUtil.commitTransaction(); + numberOfAffectedEntities.incrementAndGet(); + numberOfUpdatedEntities.incrementAndGet(); + } + + @Override + protected void doOperation() { + log("Running " + getClass().getSimpleName() + "..."); + log("Preview: " + isPreview()); + + Cursor cursor = readPositionOfCursorFromFile().orElse(null); + if (cursor == null) { + log("Start from the beginning"); + } else { + log("Start from cursor position: " + cursor.toUrlSafe()); + } + + boolean shouldContinue = true; + while (shouldContinue) { + shouldContinue = false; + Query filterQueryKeys = getFilterQuery().limit(BATCH_SIZE); + if (cursor != null) { + filterQueryKeys = filterQueryKeys.startAt(cursor); + } + QueryResults iterator; + + iterator = filterQueryKeys.iterator(); + + while (iterator.hasNext()) { + shouldContinue = true; + + doMigration(iterator.next()); + + numberOfScannedKey.incrementAndGet(); + } + + if (shouldContinue) { + cursor = iterator.getCursorAfter(); + savePositionOfCursorToFile(cursor); + log(String.format("Cursor Position: %s", cursor.toUrlSafe())); + log(String.format("Number Of Entity Key Scanned: %d", numberOfScannedKey.get())); + log(String.format("Number Of Entity affected: %d", numberOfAffectedEntities.get())); + log(String.format("Number Of Entity updated: %d", numberOfUpdatedEntities.get())); + } + } + + deleteCursorPositionFile(); + log(isPreview() ? "Preview Completed!" : "Migration Completed!"); + log("Total number of entities: " + numberOfScannedKey.get()); + log("Number of affected entities: " + numberOfAffectedEntities.get()); + log("Number of updated entities: " + numberOfUpdatedEntities.get()); + } + + /** + * Saves the cursor position to a file so it can be used in the next run. + */ + private void savePositionOfCursorToFile(Cursor cursor) { + try { + FileHelper.saveFile( + BASE_LOG_URI + this.getClass().getSimpleName() + ".cursor", cursor.toUrlSafe()); + } catch (IOException e) { + logError("Fail to save cursor position " + e.getMessage()); + } + } + + /** + * Reads the cursor position from the saved file. + * + * @return cursor if the file can be properly decoded. + */ + private Optional readPositionOfCursorFromFile() { + try { + String cursorPosition = FileHelper.readFile(BASE_LOG_URI + this.getClass().getSimpleName() + ".cursor"); + return Optional.of(Cursor.fromUrlSafe(cursorPosition)); + } catch (IOException | IllegalArgumentException e) { + return Optional.empty(); + } + } + + /** + * Deletes the cursor position file. + */ + private void deleteCursorPositionFile() { + FileHelper.deleteFile(BASE_LOG_URI + this.getClass().getSimpleName() + ".cursor"); + } + + /** + * Logs a comment. + */ + protected void log(String logLine) { + System.out.println(String.format("%s %s", getLogPrefix(), logLine)); + + Path logPath = Paths.get(BASE_LOG_URI + this.getClass().getSimpleName() + ".log"); + try (OutputStream logFile = Files.newOutputStream(logPath, + StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.APPEND)) { + logFile.write((logLine + System.lineSeparator()).getBytes(Const.ENCODING)); + } catch (Exception e) { + System.err.println("Error writing log line: " + logLine); + System.err.println(e.getMessage()); + } + } + + /** + * Logs an error and persists it to the disk. + */ + protected void logError(String logLine) { + System.err.println(logLine); + + log("[ERROR]" + logLine); + } + +} diff --git a/src/client/java/teammates/client/scripts/sql/VerifyAccountRequestAttributes.java b/src/client/java/teammates/client/scripts/sql/VerifyAccountRequestAttributes.java index 28a943c6db0..dee1080ec6f 100644 --- a/src/client/java/teammates/client/scripts/sql/VerifyAccountRequestAttributes.java +++ b/src/client/java/teammates/client/scripts/sql/VerifyAccountRequestAttributes.java @@ -29,25 +29,20 @@ public static void main(String[] args) { @Override public boolean equals(teammates.storage.sqlentity.AccountRequest sqlEntity, AccountRequest datastoreEntity) { if (datastoreEntity != null) { - // UUID for account is not checked, as datastore ID is email%institute - if (!sqlEntity.getName().equals(datastoreEntity.getName())) { - return false; - } - if (!sqlEntity.getEmail().equals(datastoreEntity.getEmail())) { - return false; - } - if (!sqlEntity.getInstitute().equals(datastoreEntity.getInstitute())) { - return false; - } - // only need to check getRegisteredAt() as the other fields must not be null. - if (sqlEntity.getRegisteredAt() == null) { - if (datastoreEntity.getRegisteredAt() != null) { - return false; - } - } else if (!sqlEntity.getRegisteredAt().equals(datastoreEntity.getRegisteredAt())) { - return false; + boolean matchingCreatedAtTimestamp = false; + + if (sqlEntity.getCreatedAt() == null || datastoreEntity.getCreatedAt() == null) { + matchingCreatedAtTimestamp = sqlEntity.getCreatedAt() == datastoreEntity.getCreatedAt(); + } else { + matchingCreatedAtTimestamp = sqlEntity.getCreatedAt().equals(datastoreEntity.getCreatedAt()); } - return true; + + // UUID for account is not checked, as datastore ID is email%institute + return sqlEntity.getName().equals(datastoreEntity.getName()) + && sqlEntity.getEmail().equals(datastoreEntity.getEmail()) + && sqlEntity.getInstitute().equals(datastoreEntity.getInstitute()) + && sqlEntity.getRegisteredAt().equals(datastoreEntity.getRegisteredAt()) + && matchingCreatedAtTimestamp; } else { return false; } diff --git a/src/client/java/teammates/client/scripts/sql/VerifyNotificationAttributes.java b/src/client/java/teammates/client/scripts/sql/VerifyNotificationAttributes.java index d7daf1a253c..44ec97d3b90 100644 --- a/src/client/java/teammates/client/scripts/sql/VerifyNotificationAttributes.java +++ b/src/client/java/teammates/client/scripts/sql/VerifyNotificationAttributes.java @@ -26,6 +26,14 @@ protected String generateID(teammates.storage.sqlentity.Notification sqlEntity) public boolean equals(teammates.storage.sqlentity.Notification sqlEntity, Notification datastoreEntity) { try { UUID otherUuid = UUID.fromString(datastoreEntity.getNotificationId()); + boolean matchingCreatedAtTimestamp = false; + + if (sqlEntity.getCreatedAt() == null || datastoreEntity.getCreatedAt() == null) { + matchingCreatedAtTimestamp = sqlEntity.getCreatedAt() == datastoreEntity.getCreatedAt(); + } else { + matchingCreatedAtTimestamp = sqlEntity.getCreatedAt().equals(datastoreEntity.getCreatedAt()); + } + return sqlEntity.getId().equals(otherUuid) && sqlEntity.getStartTime().equals(datastoreEntity.getStartTime()) && sqlEntity.getEndTime().equals(datastoreEntity.getEndTime()) @@ -33,6 +41,7 @@ public boolean equals(teammates.storage.sqlentity.Notification sqlEntity, Notifi && sqlEntity.getTargetUser().equals(datastoreEntity.getTargetUser()) && sqlEntity.getTitle().equals(datastoreEntity.getTitle()) && sqlEntity.getMessage().equals(datastoreEntity.getMessage()) + && matchingCreatedAtTimestamp && sqlEntity.isShown() == datastoreEntity.isShown(); } catch (IllegalArgumentException iae) { return false;