Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Performance optimizations when bulk loading large amounts of timestamps #2194

Merged
merged 4 commits into from
Nov 2, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 11 additions & 15 deletions src/main/java/com/microsoft/sqlserver/jdbc/IOBuffer.java
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import java.sql.Timestamp;
import java.text.MessageFormat;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.OffsetTime;
import java.util.ArrayList;
Expand Down Expand Up @@ -3755,29 +3756,24 @@ void writeSmalldatetime(String value) throws SQLServerException {
writeShort((short) minutesSinceMidnight);
}

void writeDatetime(String value) throws SQLServerException {
GregorianCalendar calendar = initializeCalender(TimeZone.getDefault());
long utcMillis; // Value to which the calendar is to be set (in milliseconds 1/1/1970 00:00:00 GMT)
void writeDatetime(java.sql.Timestamp dateValue) throws SQLServerException {
LocalDateTime ldt = dateValue.toLocalDateTime();
int subSecondNanos;
java.sql.Timestamp timestampValue = java.sql.Timestamp.valueOf(value);
utcMillis = timestampValue.getTime();
subSecondNanos = timestampValue.getNanos();
subSecondNanos = ldt.getNano();

// Load the calendar with the desired value
calendar.setTimeInMillis(utcMillis);

// Number of days there have been since the SQL Base Date.
// These are based on SQL Server algorithms
int daysSinceSQLBaseDate = DDC.daysSinceBaseDate(calendar.get(Calendar.YEAR),
calendar.get(Calendar.DAY_OF_YEAR), TDS.BASE_YEAR_1900);
int daysSinceSQLBaseDate = DDC.daysSinceBaseDate(ldt.getYear(),
ldt.getDayOfYear(), TDS.BASE_YEAR_1900);

// Number of milliseconds since midnight of the current day.
int millisSinceMidnight = (subSecondNanos + Nanos.PER_MILLISECOND / 2) / Nanos.PER_MILLISECOND + // Millis into
// the current
// second
1000 * calendar.get(Calendar.SECOND) + // Seconds into the current minute
60 * 1000 * calendar.get(Calendar.MINUTE) + // Minutes into the current hour
60 * 60 * 1000 * calendar.get(Calendar.HOUR_OF_DAY); // Hours into the current day
// the current
// second
1000 * ldt.getSecond() + // Seconds into the current minute
60 * 1000 * ldt.getMinute() + // Minutes into the current hour
60 * 60 * 1000 * ldt.getHour(); // Hours into the current day

// The last millisecond of the current day is always rounded to the first millisecond
// of the next day because DATETIME is only accurate to 1/300th of a second.
Expand Down
19 changes: 12 additions & 7 deletions src/main/java/com/microsoft/sqlserver/jdbc/SQLServerBulkCopy.java
Original file line number Diff line number Diff line change
Expand Up @@ -2461,7 +2461,12 @@ else if (null != sourceCryptoMeta) {
case DATETIME:
if (bulkNullable)
tdsWriter.writeByte((byte) 0x08);
tdsWriter.writeDatetime(colValue.toString());

if (colValue instanceof java.sql.Timestamp) {
tdsWriter.writeDatetime((java.sql.Timestamp) colValue);
} else {
tdsWriter.writeDatetime(java.sql.Timestamp.valueOf(colValue.toString()));
}
break;
default: // DATETIME2
if (2 >= bulkScale)
Expand Down Expand Up @@ -2695,15 +2700,15 @@ private void writeSqlVariant(TDSWriter tdsWriter, Object colValue, ResultSet sou
tdsWriter.writeTime((java.sql.Timestamp) colValue, timeBulkScale);
break;

case DATETIME8:
writeBulkCopySqlVariantHeader(10, TDSType.DATETIME8.byteValue(), (byte) 0, tdsWriter);
tdsWriter.writeDatetime(colValue.toString());
break;

case DATETIME4:
// when the type is ambiguous, we write to bigger type
case DATETIME8:
writeBulkCopySqlVariantHeader(10, TDSType.DATETIME8.byteValue(), (byte) 0, tdsWriter);
tdsWriter.writeDatetime(colValue.toString());
if (colValue instanceof java.sql.Timestamp) {
tdsWriter.writeDatetime((java.sql.Timestamp) colValue);
} else {
tdsWriter.writeDatetime(java.sql.Timestamp.valueOf(colValue.toString()));
}
break;

case DATETIME2N:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Timestamp;
import java.sql.Types;

import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Tag;
Expand All @@ -16,15 +18,33 @@
import org.junit.runner.RunWith;

import com.microsoft.sqlserver.jdbc.ComparisonUtil;
import com.microsoft.sqlserver.jdbc.RandomData;
import com.microsoft.sqlserver.jdbc.RandomUtil;
import com.microsoft.sqlserver.jdbc.SQLServerBulkCopy;
import com.microsoft.sqlserver.jdbc.SQLServerBulkCopyOptions;
import com.microsoft.sqlserver.jdbc.TestUtils;
import com.microsoft.sqlserver.testframework.AbstractSQLGenerator;
import com.microsoft.sqlserver.testframework.AbstractTest;
import com.microsoft.sqlserver.testframework.Constants;
import com.microsoft.sqlserver.testframework.DBConnection;
import com.microsoft.sqlserver.testframework.DBStatement;
import com.microsoft.sqlserver.testframework.DBTable;
import com.microsoft.sqlserver.testframework.PrepUtil;

import javax.sql.RowSetMetaData;
import javax.sql.rowset.CachedRowSet;
import javax.sql.rowset.RowSetFactory;
import javax.sql.rowset.RowSetMetaDataImpl;
import javax.sql.rowset.RowSetProvider;

import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;


@RunWith(JUnitPlatform.class)
public class BulkCopyAllTypesTest extends AbstractTest {
Expand Down Expand Up @@ -98,6 +118,71 @@ private void terminateVariation() throws SQLException {
try (Statement stmt = connection.createStatement()) {
TestUtils.dropTableIfExists(tableSrc.getEscapedTableName(), stmt);
TestUtils.dropTableIfExists(tableDest.getEscapedTableName(), stmt);
TestUtils.dropTableIfExists(dateTimeTestTable, stmt);
}
}

private static final int DATETIME_COL_COUNT = 2;
private static final int DATETIME_ROW_COUNT = 1;
private static final String dateTimeTestTable =
AbstractSQLGenerator.escapeIdentifier(RandomUtil.getIdentifier("bulkCopyTimestampTest"));

@Test
public void testBulkCopyTimestamp() throws SQLException {
List<Timestamp> timeStamps = new ArrayList<>();
try (Connection con = getConnection(); Statement stmt = connection.createStatement()) {
String colSpec = IntStream.range(1, DATETIME_COL_COUNT + 1).mapToObj(x -> String.format("c%d datetime", x)).collect(
Collectors.joining(","));
String sql1 = String.format("create table %s (%s)", dateTimeTestTable, colSpec);
stmt.execute(sql1);

RowSetFactory rsf = RowSetProvider.newFactory();
CachedRowSet crs = rsf.createCachedRowSet();
RowSetMetaData rsmd = new RowSetMetaDataImpl();
rsmd.setColumnCount(DATETIME_COL_COUNT);

for (int i = 1; i <= DATETIME_COL_COUNT; i++) {
rsmd.setColumnName(i, String.format("c%d", i));
rsmd.setColumnType(i, Types.TIMESTAMP);
}
crs.setMetaData(rsmd);

for (int i = 0; i < DATETIME_COL_COUNT; i++) {
timeStamps.add(RandomData.generateDatetime(false));
}

for (int ri = 0; ri < DATETIME_ROW_COUNT; ri++) {
crs.moveToInsertRow();

for (int i = 1; i <= DATETIME_COL_COUNT; i++) {
crs.updateTimestamp(i, timeStamps.get(i - 1));
}
crs.insertRow();
}
crs.moveToCurrentRow();

try (SQLServerBulkCopy bcOperation = new SQLServerBulkCopy(con)) {
SQLServerBulkCopyOptions bcOptions = new SQLServerBulkCopyOptions();
bcOptions.setBatchSize(5000);
bcOperation.setDestinationTableName(dateTimeTestTable);
bcOperation.setBulkCopyOptions(bcOptions);
bcOperation.writeToServer(crs);
}

try (ResultSet rs = stmt.executeQuery("select * from " + dateTimeTestTable)) {
assertTrue(rs.next());

for (int i = 1; i <= DATETIME_COL_COUNT; i++) {
long expectedTimestamp = getTime(timeStamps.get(i - 1));
long actualTimestamp = getTime(rs.getTimestamp(i));

assertEquals(expectedTimestamp, actualTimestamp);
}
}
}
}

private static long getTime(Timestamp time) {
return (3 * time.getTime() + 5) / 10;
}
}
Loading