Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Performance optimizations when bulk loading large amounts of timestamps #2194

Merged
merged 4 commits into from
Nov 2, 2023
Merged

Performance optimizations when bulk loading large amounts of timestamps #2194

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
34f6010
Performance optimizations when bulk loading large amounts of timestamps
hannes92 Aug 15, 2023
df938e1
Added test case for bulk loading many rows of timestamps
hannes92 Aug 16, 2023
73c3d88
Adjusted core per review remarks
hannes92 Sep 28, 2023
74db912
Adjusted code per review
hannes92 Oct 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 22 additions & 56 deletions src/main/java/com/microsoft/sqlserver/jdbc/IOBuffer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,28 +5,17 @@

package com.microsoft.sqlserver.jdbc;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.Reader;
import java.io.Serializable;
import java.io.UnsupportedEncodingException;
import com.microsoft.sqlserver.jdbc.SQLServerConnection.FedAuthTokenCommand;
import com.microsoft.sqlserver.jdbc.dataclassification.SensitivityClassification;

import javax.net.SocketFactory;
import javax.net.ssl.*;
import java.io.*;
lilgreenbird marked this conversation as resolved.
Show resolved Hide resolved
import java.lang.reflect.InvocationTargetException;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.math.RoundingMode;
import java.net.Inet4Address;
import java.net.Inet6Address;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.net.SocketAddress;
import java.net.SocketException;
import java.net.SocketTimeoutException;
import java.net.*;
import java.nio.Buffer;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
Expand All @@ -40,22 +29,11 @@
import java.sql.Timestamp;
import java.text.MessageFormat;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.OffsetTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.GregorianCalendar;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.*;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
import java.util.SimpleTimeZone;
import java.util.TimeZone;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
Expand All @@ -68,18 +46,6 @@
import java.util.logging.Level;
import java.util.logging.Logger;

import javax.net.SocketFactory;
import javax.net.ssl.KeyManager;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLParameters;
import javax.net.ssl.SSLSocket;
import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactory;
import javax.net.ssl.X509TrustManager;

import com.microsoft.sqlserver.jdbc.SQLServerConnection.FedAuthTokenCommand;
import com.microsoft.sqlserver.jdbc.dataclassification.SensitivityClassification;


final class TDS {
// application protocol
Expand Down Expand Up @@ -3756,28 +3722,28 @@ void writeSmalldatetime(String value) throws SQLServerException {
}

void writeDatetime(String value) throws SQLServerException {
GregorianCalendar calendar = initializeCalender(TimeZone.getDefault());
writeDatetime(java.sql.Timestamp.valueOf(value));
}
hannes92 marked this conversation as resolved.
Show resolved Hide resolved

void writeDatetime(java.sql.Timestamp dateValue) throws SQLServerException {
LocalDateTime ldt = dateValue.toLocalDateTime();
long utcMillis; // Value to which the calendar is to be set (in milliseconds 1/1/1970 00:00:00 GMT)
hannes92 marked this conversation as resolved.
Show resolved Hide resolved
int subSecondNanos;
java.sql.Timestamp timestampValue = java.sql.Timestamp.valueOf(value);
utcMillis = timestampValue.getTime();
subSecondNanos = timestampValue.getNanos();
subSecondNanos = ldt.getNano();

// Load the calendar with the desired value
calendar.setTimeInMillis(utcMillis);

// Number of days there have been since the SQL Base Date.
// These are based on SQL Server algorithms
int daysSinceSQLBaseDate = DDC.daysSinceBaseDate(calendar.get(Calendar.YEAR),
calendar.get(Calendar.DAY_OF_YEAR), TDS.BASE_YEAR_1900);
int daysSinceSQLBaseDate = DDC.daysSinceBaseDate(ldt.getYear(),
ldt.getDayOfYear(), TDS.BASE_YEAR_1900);

// Number of milliseconds since midnight of the current day.
int millisSinceMidnight = (subSecondNanos + Nanos.PER_MILLISECOND / 2) / Nanos.PER_MILLISECOND + // Millis into
// the current
// second
1000 * calendar.get(Calendar.SECOND) + // Seconds into the current minute
60 * 1000 * calendar.get(Calendar.MINUTE) + // Minutes into the current hour
60 * 60 * 1000 * calendar.get(Calendar.HOUR_OF_DAY); // Hours into the current day
// the current
// second
1000 * ldt.getSecond() + // Seconds into the current minute
60 * 1000 * ldt.getMinute() + // Minutes into the current hour
60 * 60 * 1000 * ldt.getHour(); // Hours into the current day

// The last millisecond of the current day is always rounded to the first millisecond
// of the next day because DATETIME is only accurate to 1/300th of a second.
Expand Down
6 changes: 5 additions & 1 deletion src/main/java/com/microsoft/sqlserver/jdbc/SQLServerBulkCopy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2461,7 +2461,11 @@ else if (null != sourceCryptoMeta) {
case DATETIME:
if (bulkNullable)
tdsWriter.writeByte((byte) 0x08);
tdsWriter.writeDatetime(colValue.toString());

if (colValue instanceof java.sql.Timestamp)
tdsWriter.writeDatetime((java.sql.Timestamp)colValue);
else
tdsWriter.writeDatetime(colValue.toString());
hannes92 marked this conversation as resolved.
Show resolved Hide resolved
break;
default: // DATETIME2
if (2 >= bulkScale)
Expand Down
111 changes: 111 additions & 0 deletions src/test/java/com/microsoft/sqlserver/jdbc/bulkCopy/BulkCopyTimestampTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
package com.microsoft.sqlserver.jdbc.bulkCopy;
lilgreenbird marked this conversation as resolved.
Show resolved Hide resolved

import com.microsoft.sqlserver.jdbc.*;
lilgreenbird marked this conversation as resolved.
Show resolved Hide resolved
import com.microsoft.sqlserver.testframework.AbstractSQLGenerator;
import com.microsoft.sqlserver.testframework.AbstractTest;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.platform.runner.JUnitPlatform;
import org.junit.runner.RunWith;

import javax.sql.RowSetMetaData;
import javax.sql.rowset.CachedRowSet;
import javax.sql.rowset.RowSetFactory;
import javax.sql.rowset.RowSetMetaDataImpl;
import javax.sql.rowset.RowSetProvider;
import java.sql.*;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;


@RunWith(JUnitPlatform.class)
public class BulkCopyTimestampTest extends AbstractTest {

public static final int COLUMN_COUNT = 16;
public static final int ROW_COUNT = 10000;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it's not really necessary to do this that many times, just make sure this is run at last once cover the newly added code

private static final String tableName =
AbstractSQLGenerator.escapeIdentifier(RandomUtil.getIdentifier("bulkCopyTimestampTest"));

@Test
public void testBulkCopyTimestamp() throws SQLException {
List<Timestamp> timeStamps = new ArrayList<>();
try (Connection con = getConnection(); Statement stmt = connection.createStatement()) {
RowSetFactory rsf = RowSetProvider.newFactory();
CachedRowSet crs = rsf.createCachedRowSet();
RowSetMetaData rsmd = new RowSetMetaDataImpl();
rsmd.setColumnCount(COLUMN_COUNT);

for (int i = 1; i <= COLUMN_COUNT; i++) {
rsmd.setColumnName(i, String.format("c%d", i));
rsmd.setColumnType(i, Types.TIMESTAMP);
}

crs.setMetaData(rsmd);


for (int i = 0; i < COLUMN_COUNT; i++) {
timeStamps.add(RandomData.generateDatetime(false));
}

for (int ri = 0; ri < ROW_COUNT; ri++) {
crs.moveToInsertRow();

for (int i = 1; i <= COLUMN_COUNT; i++) {
crs.updateTimestamp(i, timeStamps.get(i - 1));

lilgreenbird marked this conversation as resolved.
Show resolved Hide resolved
}

crs.insertRow();
}

crs.moveToCurrentRow();

try (SQLServerBulkCopy bcOperation = new SQLServerBulkCopy(con)) {
SQLServerBulkCopyOptions bcOptions = new SQLServerBulkCopyOptions();
bcOptions.setBatchSize(5000);
bcOperation.setDestinationTableName(tableName);
bcOperation.setBulkCopyOptions(bcOptions);
bcOperation.writeToServer(crs);
}

try (ResultSet rs = stmt.executeQuery("select * from " + tableName)) {
assertTrue(rs.next());

for (int i = 1; i <= COLUMN_COUNT; i++) {
long expectedTimestamp = getTime(timeStamps.get(i - 1));
long actualTimestamp = getTime(rs.getTimestamp(i));

assertEquals(expectedTimestamp, actualTimestamp);
}
}
}
}

private static long getTime(Timestamp time) {
return (3 * time.getTime() + 5) / 10;
}

@BeforeAll
public static void testSetup() throws Exception {
setConnection();

try (Statement stmt = connection.createStatement()) {
String colSpec = IntStream.range(1, COLUMN_COUNT + 1).mapToObj(x -> String.format("c%d datetime", x)).collect(Collectors.joining(","));
String sql1 = String.format("create table %s (%s)", tableName, colSpec);
stmt.execute(sql1);
}
}

@AfterAll
public static void terminateVariation() throws SQLException {
try (Statement stmt = connection.createStatement()) {
TestUtils.dropTableIfExists(tableName, stmt);
}
}
}