Skip to content

Commit

Permalink
Merge pull request #60 from Azure/dev
Browse files Browse the repository at this point in the history
Client request properties and ResultSet ingestion
  • Loading branch information
tamirkamara authored Dec 19, 2018
2 parents 75cd3c9 + 6711a00 commit 1217310
Show file tree
Hide file tree
Showing 22 changed files with 693 additions and 84 deletions.
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Microsoft Azure Kusto (Azure Data Explorer) SDK for Java

master: [![Build Status](https://travis-ci.org/Azure/azure-kusto-java.svg?branch=master)](https://travis-ci.org/Azure/azure-kusto-java)
master: [![Build Status](https://travis-ci.org/Azure/azure-kusto-java.svg)](https://travis-ci.org/Azure/azure-kusto-java)
dev: [![Build Status](https://travis-ci.org/Azure/azure-kusto-java.svg?branch=dev)](https://travis-ci.org/Azure/azure-kusto-java)

This is the Microsoft Azure Kusto client library which allows communication with Kusto to bring data in (ingest) and query information already stored in the database.
Expand Down
13 changes: 10 additions & 3 deletions data/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,9 @@
<configuration>
<archive>
<manifest>
<mainClass>com.microsoft.azure.kusto.ingest.Client</mainClass>
<mainClass>com.microsoft.azure.kusto.data.Client</mainClass>
<addDefaultImplementationEntries>true</addDefaultImplementationEntries>
<addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>
</manifest>
</archive>
<descriptorRefs>
Expand All @@ -53,7 +55,7 @@
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>adal4j</artifactId>
<version>1.6.0</version>
<version>1.6.3</version>
</dependency>

<!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient -->
Expand Down Expand Up @@ -98,7 +100,12 @@
<version>${mockito.version}</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.skyscreamer</groupId>
<artifactId>jsonassert</artifactId>
<version>1.5.0</version>
<scope>test</scope>
</dependency>
</dependencies>


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,5 @@ public interface Client {

Results execute(String database, String command) throws DataServiceException, DataClientException;

Results execute(String database, String command, ClientRequestProperties properties) throws DataServiceException, DataClientException;
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,5 +7,4 @@ public class ClientFactory {
public static Client createClient(ConnectionStringBuilder csb) throws URISyntaxException {
return new ClientImpl(csb);
}

}
44 changes: 33 additions & 11 deletions data/src/main/java/com/microsoft/azure/kusto/data/ClientImpl.java
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,15 @@
import org.json.JSONObject;

import java.net.URISyntaxException;
import java.util.concurrent.TimeUnit;

public class ClientImpl implements Client {

private static final String ADMIN_COMMANDS_PREFIX = ".";
private static final String API_VERSION = "v1";
private static final String DEFAULT_DATABASE_NAME = "NetDefaultDb";
private static final Long COMMAND_TIMEOUT_IN_MILLISECS = TimeUnit.MINUTES.toMillis(10) + TimeUnit.SECONDS.toMillis(30);
private static final Long QUERY_TIMEOUT_IN_MILLISECS = TimeUnit.MINUTES.toMillis(4) + TimeUnit.SECONDS.toMillis(30);

private AadAuthenticationHelper aadAuthenticationHelper;
private String clusterUrl;
Expand All @@ -27,31 +30,50 @@ public Results execute(String command) throws DataServiceException, DataClientEx
}

public Results execute(String database, String command) throws DataServiceException, DataClientException {
return execute(database, command, null);
}

public Results execute(String database, String command, ClientRequestProperties properties) throws DataServiceException, DataClientException {
// Argument validation:
if (StringUtils.isAnyEmpty(database, command)) {
throw new IllegalArgumentException("database or command are empty");
}

Long timeoutMs = null;

if (properties != null) {
timeoutMs = properties.getTimeoutInMilliSec();
}

String clusterEndpoint;
if (command.startsWith(ADMIN_COMMANDS_PREFIX)) {
clusterEndpoint = String.format("%s/%s/rest/mgmt", clusterUrl, API_VERSION);
if (timeoutMs == null) {
timeoutMs = COMMAND_TIMEOUT_IN_MILLISECS;
}
} else {
clusterEndpoint = String.format("%s/%s/rest/query", clusterUrl, API_VERSION);
}
return execute(database, command, clusterEndpoint);
}

private Results execute(String database, String command, String clusterEndpoint) throws DataServiceException, DataClientException {
// Argument validation:
if (StringUtils.isAnyEmpty(database, command, clusterEndpoint)) {
throw new IllegalArgumentException("database, command or clusterEndpoint are empty");
if (timeoutMs == null) {
timeoutMs = QUERY_TIMEOUT_IN_MILLISECS;
}
}

String aadAccessToken = aadAuthenticationHelper.acquireAccessToken();
String jsonString;
try {
jsonString = new JSONObject()
JSONObject json = new JSONObject()
.put("db", database)
.put("csl", command).toString();
.put("csl", command);

if (properties != null) {
json.put("properties", properties.toString());
}

jsonString = json.toString();
} catch (JSONException e) {
throw new DataClientException(clusterEndpoint, String.format(clusterEndpoint, "Error in executing command: %s, in database: %s", command, database), e);
}

return Utils.post(clusterEndpoint, aadAccessToken, jsonString);
return Utils.post(clusterEndpoint, aadAccessToken, jsonString, timeoutMs.intValue());
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
package com.microsoft.azure.kusto.data;

import org.json.JSONObject;

import java.util.HashMap;

/*
* Kusto supports attaching various properties to client requests (such as queries and control commands).
* Such properties may be used to provide additional information to Kusto (for example, for the purpose of correlating client/service interaction),
* may affect what limits and policies get applied to the request, and much more.
* For a complete list of available client request properties
* check out https://docs.microsoft.com/en-us/azure/kusto/api/netfx/request-properties#list-of-clientrequestproperties
*/
public class ClientRequestProperties {
private static final String OPTIONS_KEY = "Options";
private static final String OPTION_SERVER_TIMEOUT = "servertimeout";
private HashMap<String, Object> properties;
private HashMap<String, Object> options;

public ClientRequestProperties() {
properties = new HashMap<>();
options = new HashMap<>();
properties.put(OPTIONS_KEY, options);
}

public void setOption(String name, Object value) {
options.put(name, value);
}

public Object getOption(String name) {
return options.get(name);
}

public void removeOption(String name) {
options.remove(name);
}

public void clearOptions() {
options.clear();
}

public Long getTimeoutInMilliSec() {
return (Long) getOption(OPTION_SERVER_TIMEOUT);
}

public void setTimeoutInMilliSec(Long timeoutInMs) {
options.put(OPTION_SERVER_TIMEOUT, timeoutInMs);
}

JSONObject toJson() {
return new JSONObject(properties);
}

public String toString() {
return toJson().toString();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ private ConnectionStringBuilder(String resourceUri)
privateKey = null;
}

private static ConnectionStringBuilder createWithAadUserCredentials(String resourceUri,
public static ConnectionStringBuilder createWithAadUserCredentials(String resourceUri,
String username,
String password,
String authorityId)
Expand Down
31 changes: 20 additions & 11 deletions data/src/main/java/com/microsoft/azure/kusto/data/Results.java
Original file line number Diff line number Diff line change
Expand Up @@ -8,21 +8,30 @@ public class Results {
private HashMap<String, String> columnNameToType;
private ArrayList<ArrayList<String>> values;

public HashMap<String, Integer> getColumnNameToIndex() { return columnNameToIndex; }
public Results(HashMap<String, Integer> columnNameToIndex, HashMap<String, String> columnNameToType,
ArrayList<ArrayList<String>> values) {
this.columnNameToIndex = columnNameToIndex;
this.columnNameToType = columnNameToType;
this.values = values;
}

public HashMap<String, String> getColumnNameToType() { return columnNameToType; }
public HashMap<String, Integer> getColumnNameToIndex() {
return columnNameToIndex;
}

public Integer getIndexByColumnName(String columnName) { return columnNameToIndex.get(columnName); }
public HashMap<String, String> getColumnNameToType() {
return columnNameToType;
}

public String getTypeByColumnName(String columnName) { return columnNameToType.get(columnName); }
public Integer getIndexByColumnName(String columnName) {
return columnNameToIndex.get(columnName);
}

public ArrayList<ArrayList<String>> getValues() { return values; }
public String getTypeByColumnName(String columnName) {
return columnNameToType.get(columnName);
}

public Results(HashMap<String, Integer> columnNameToIndex, HashMap<String, String> columnNameToType,
ArrayList<ArrayList<String>> values)
{
this.columnNameToIndex = columnNameToIndex;
this.columnNameToType = columnNameToType;
this.values = values;
public ArrayList<ArrayList<String>> getValues() {
return values;
}
}
27 changes: 23 additions & 4 deletions data/src/main/java/com/microsoft/azure/kusto/data/Utils.java
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,16 @@
import com.microsoft.azure.kusto.data.exceptions.DataClientException;
import com.microsoft.azure.kusto.data.exceptions.DataServiceException;
import com.microsoft.azure.kusto.data.exceptions.DataWebException;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.StatusLine;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.json.JSONArray;
Expand All @@ -22,8 +25,16 @@

class Utils {

static Results post(String url, String aadAccessToken, String payload) throws DataServiceException, DataClientException {
HttpClient httpClient = HttpClients.createSystem();
static Results post(String url, String aadAccessToken, String payload, Integer timeoutMs) throws DataServiceException, DataClientException {

HttpClient httpClient;
if (timeoutMs != null) {
RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(timeoutMs).build();
httpClient = HttpClientBuilder.create().useSystemProperties().setDefaultRequestConfig(requestConfig).build();
} else {
httpClient = HttpClients.createSystem();
}

HttpPost httpPost = new HttpPost(url);

// Request parameters and other properties.
Expand All @@ -38,7 +49,15 @@ static Results post(String url, String aadAccessToken, String payload) throws Da
httpPost.addHeader("Content-Type", "application/json");
httpPost.addHeader("Accept-Encoding", "gzip,deflate");
httpPost.addHeader("Fed", "True");
httpPost.addHeader("x-ms-client-version", "Kusto.Java.Client");

String version = Utils.class.getPackage().getImplementationVersion();
String clientVersion = "Kusto.Java.Client";
if (StringUtils.isNotBlank(version)) {
clientVersion += ":" + version;
}

httpPost.addHeader("x-ms-client-version", clientVersion);
httpPost.addHeader("x-ms-client-request-id", String.format("KJC.execute;%s", java.util.UUID.randomUUID()));

try {
//Execute and get the response.
Expand Down Expand Up @@ -71,7 +90,7 @@ static Results post(String url, String aadAccessToken, String payload) throws Da
for (int i = 0; i < resultsRows.length(); i++) {
JSONArray row = resultsRows.getJSONArray(i);
ArrayList<String> rowVector = new ArrayList<>();
for(int j = 0; j < row.length(); ++j) {
for (int j = 0; j < row.length(); ++j) {
Object obj = row.get(j);
if (obj == JSONObject.NULL) {
rowVector.add(null);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
package com.microsoft.azure.kusto.data;

import org.json.JSONException;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.skyscreamer.jsonassert.JSONAssert;

import java.util.concurrent.TimeUnit;

public class ClientRequestPropertiesTest {
@Test
@DisplayName("test set/get timeout")
void timeoutSetGet() {
ClientRequestProperties props = new ClientRequestProperties();
Long expected = TimeUnit.MINUTES.toMillis(100);

// before setting value should be null
Assertions.assertEquals(null, props.getTimeoutInMilliSec());

props.setTimeoutInMilliSec(expected);
Assertions.assertEquals(expected, props.getTimeoutInMilliSec());
}

@Test
@DisplayName("test ClientRequestProperties toString")
void propertiesToString() throws JSONException {
ClientRequestProperties props = new ClientRequestProperties();
props.setOption("a", 1);
props.setOption("b", "hello");

JSONAssert.assertEquals("{\"Options\": {\"a\":1, \"b\":\"hello\"}}", props.toString(), false);
}
}
7 changes: 7 additions & 0 deletions data/src/test/resources/simplelogger.properties
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
# SLF4J's SimpleLogger configuration file
# Simple implementation of Logger that sends all enabled log messages, for all defined loggers, to System.err.

# Default logging detail level for all instances of SimpleLogger.
# Must be one of ("trace", "debug", "info", "warn", or "error").
# If not specified, defaults to "info".
org.slf4j.simpleLogger.log.com.microsoft.azure.kusto=debug
12 changes: 12 additions & 0 deletions ingest/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,18 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
<archive>
<manifest>
<addDefaultImplementationEntries>true</addDefaultImplementationEntries>
<addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>
</manifest>
</archive>
</configuration>
</plugin>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>${maven-surefire-plugin.version}</version>
Expand Down
Loading

0 comments on commit 1217310

Please sign in to comment.