Skip to content

Commit

Permalink
[CN-399] [BACKPORT] Change Kubernetes client behavior for token refre…
Browse files Browse the repository at this point in the history 
…sh (#22394)

Kubernetes token which is read from the file was static in the
beginning. With the recent changes, it is time-bound and its content
is periodically refreshed.

To incorporate new behavior:
* Introduced a new token reader which gets the value by reading the file
  every time.
* Introduced a second token reader which imitates the original
  behavior.

Co-authored-by: Hakan Memisoglu <hakanmemisoglu@gmail.com>
  • Loading branch information
@hasancelik @hakanmemisoglu
hasancelik and hakanmemisoglu authored Oct 5, 2022
1 parent 666e7d5 commit 9a682d4
Show file tree
Hide file tree
Showing 8 changed files with 178 additions and 35 deletions.
32 changes: 32 additions & 0 deletions hazelcast/src/main/java/com/hazelcast/kubernetes/FileReaderTokenProvider.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
/*
* Copyright (c) 2008-2022, Hazelcast, Inc. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.hazelcast.kubernetes;

class FileReaderTokenProvider implements KubernetesTokenProvider {
private final String tokenPath;
private final KubernetesConfig.FileContentsReader fileContentsReader;

FileReaderTokenProvider(String tokenPath) {
this.tokenPath = tokenPath;
this.fileContentsReader = new KubernetesConfig.DefaultFileContentsReader();
}

@Override
public String getToken() {
return fileContentsReader.readFileContents(tokenPath);
}
}
2 changes: 1 addition & 1 deletion hazelcast/src/main/java/com/hazelcast/kubernetes/HazelcastKubernetesDiscoveryStrategy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ final class HazelcastKubernetesDiscoveryStrategy
}

private static KubernetesClient buildKubernetesClient(KubernetesConfig config) {
return new KubernetesClient(config.getNamespace(), config.getKubernetesMasterUrl(), config.getKubernetesApiToken(),
return new KubernetesClient(config.getNamespace(), config.getKubernetesMasterUrl(), config.getTokenProvider(),
config.getKubernetesCaCertificate(), config.getKubernetesApiRetries(), config.getExposeExternallyMode(),
config.isUseNodeNameAsExternalAddress(), config.getServicePerPodLabelName(), config.getServicePerPodLabelValue());
}
Expand Down
31 changes: 19 additions & 12 deletions hazelcast/src/main/java/com/hazelcast/kubernetes/KubernetesClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,13 +51,11 @@ class KubernetesClient {

private static final List<String> NON_RETRYABLE_KEYWORDS = asList(
"\"reason\":\"Forbidden\"",
"\"reason\":\"Unauthorized\"",
"\"reason\":\"NotFound\"",
"Failure in generating SSLSocketFactory");

private final String namespace;
private final String kubernetesMaster;
private final String apiToken;
private final String caCertificate;
private final int retries;
private final KubernetesApiProvider apiProvider;
Expand All @@ -66,15 +64,18 @@ class KubernetesClient {
private final String servicePerPodLabelName;
private final String servicePerPodLabelValue;

private final KubernetesTokenProvider tokenProvider;

private boolean isNoPublicIpAlreadyLogged;
private boolean isKnownExceptionAlreadyLogged;

KubernetesClient(String namespace, String kubernetesMaster, String apiToken, String caCertificate, int retries,
ExposeExternallyMode exposeExternallyMode, boolean useNodeNameAsExternalAddress,
String servicePerPodLabelName, String servicePerPodLabelValue) {
KubernetesClient(String namespace, String kubernetesMaster, KubernetesTokenProvider tokenProvider,
String caCertificate, int retries, ExposeExternallyMode exposeExternallyMode,
boolean useNodeNameAsExternalAddress, String servicePerPodLabelName,
String servicePerPodLabelValue) {
this.namespace = namespace;
this.kubernetesMaster = kubernetesMaster;
this.apiToken = apiToken;
this.tokenProvider = tokenProvider;
this.caCertificate = caCertificate;
this.retries = retries;
this.exposeExternallyMode = exposeExternallyMode;
Expand All @@ -84,13 +85,13 @@ class KubernetesClient {
this.apiProvider = buildKubernetesApiUrlProvider();
}

KubernetesClient(String namespace, String kubernetesMaster, String apiToken, String caCertificate, int retries,
ExposeExternallyMode exposeExternallyMode, boolean useNodeNameAsExternalAddress,
String servicePerPodLabelName, String servicePerPodLabelValue,
KubernetesApiProvider apiProvider) {
KubernetesClient(String namespace, String kubernetesMaster, KubernetesTokenProvider tokenProvider,
String caCertificate, int retries, ExposeExternallyMode exposeExternallyMode,
boolean useNodeNameAsExternalAddress, String servicePerPodLabelName,
String servicePerPodLabelValue, KubernetesApiProvider apiProvider) {
this.namespace = namespace;
this.kubernetesMaster = kubernetesMaster;
this.apiToken = apiToken;
this.tokenProvider = tokenProvider;
this.caCertificate = caCertificate;
this.retries = retries;
this.exposeExternallyMode = exposeExternallyMode;
Expand Down Expand Up @@ -221,6 +222,11 @@ String nodeName(String podName) {
return extractNodeName(callGet(podUrlString));
}

// For test purpose
boolean isKnownExceptionAlreadyLogged() {
return isKnownExceptionAlreadyLogged;
}

private static List<Endpoint> parsePodsList(JsonObject podsListJson) {
List<Endpoint> addresses = new ArrayList<>();

Expand Down Expand Up @@ -461,7 +467,8 @@ private static List<Endpoint> createEndpoints(List<Endpoint> endpoints, Map<Endp
*/
private JsonObject callGet(final String urlString) {
return RetryUtils.retry(() -> Json
.parse(RestClient.create(urlString).withHeader("Authorization", String.format("Bearer %s", apiToken))
.parse(RestClient.create(urlString)
.withHeader("Authorization", String.format("Bearer %s", tokenProvider.getToken()))
.withCaCertificates(caCertificate)
.get()
.getBody())
Expand Down
28 changes: 14 additions & 14 deletions hazelcast/src/main/java/com/hazelcast/kubernetes/KubernetesConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
@SuppressWarnings({"checkstyle:npathcomplexity", "checkstyle:cyclomaticcomplexity", "checkstyle:methodcount"})
final class KubernetesConfig {
private static final String DEFAULT_MASTER_URL = "https://kubernetes.default.svc";
private static final String DEFAULT_TOKEN_PATH = "/var/run/secrets/kubernetes.io/serviceaccount/token";
private static final int DEFAULT_SERVICE_DNS_TIMEOUT_SECONDS = 5;
private static final int DEFAULT_KUBERNETES_API_RETRIES = 3;

Expand All @@ -74,13 +75,14 @@ final class KubernetesConfig {
private final String servicePerPodLabelValue;
private final int kubernetesApiRetries;
private final String kubernetesMasterUrl;
private final String kubernetesApiToken;
private final String kubernetesCaCertificate;

// Parameters for both DNS Lookup and Kubernetes API modes
private final int servicePort;
private final FileContentsReader fileContentsReader;

private final KubernetesTokenProvider tokenProvider;

KubernetesConfig(Map<String, Comparable> properties) {
this(properties, new DefaultFileContentsReader());
}
Expand All @@ -107,7 +109,7 @@ final class KubernetesConfig {
this.kubernetesApiRetries
= getOrDefault(properties, KUBERNETES_SYSTEM_PREFIX, KUBERNETES_API_RETIRES, DEFAULT_KUBERNETES_API_RETRIES);
this.kubernetesMasterUrl = getOrDefault(properties, KUBERNETES_SYSTEM_PREFIX, KUBERNETES_MASTER_URL, DEFAULT_MASTER_URL);
this.kubernetesApiToken = getApiToken(properties);
this.tokenProvider = buildTokenProvider(properties);
this.kubernetesCaCertificate = caCertificate(properties);
this.servicePort = getOrDefault(properties, KUBERNETES_SYSTEM_PREFIX, SERVICE_PORT, 0);
this.namespace = getNamespaceWithFallbacks(properties, KUBERNETES_SYSTEM_PREFIX, NAMESPACE);
Expand Down Expand Up @@ -146,12 +148,15 @@ private String getNamespaceWithFallbacks(Map<String, Comparable> properties,
return namespace;
}

private String getApiToken(Map<String, Comparable> properties) {
private KubernetesTokenProvider buildTokenProvider(Map<String, Comparable> properties) {
String apiToken = getOrNull(properties, KUBERNETES_SYSTEM_PREFIX, KUBERNETES_API_TOKEN);
KubernetesTokenProvider apiTokenProvider;
if (apiToken == null && getMode() == DiscoveryMode.KUBERNETES_API) {
apiToken = readAccountToken();
apiTokenProvider = new FileReaderTokenProvider(DEFAULT_TOKEN_PATH);
} else {
apiTokenProvider = new StaticTokenProvider(apiToken);
}
return apiToken;
return apiTokenProvider;
}

private String caCertificate(Map<String, Comparable> properties) {
Expand All @@ -162,11 +167,6 @@ private String caCertificate(Map<String, Comparable> properties) {
return caCertificate;
}

@SuppressFBWarnings("DMI_HARDCODED_ABSOLUTE_FILENAME")
private String readAccountToken() {
return fileContentsReader.readFileContents("/var/run/secrets/kubernetes.io/serviceaccount/token");
}

@SuppressFBWarnings("DMI_HARDCODED_ABSOLUTE_FILENAME")
private String readCaCertificate() {
return fileContentsReader.readFileContents("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt");
Expand Down Expand Up @@ -365,10 +365,6 @@ String getKubernetesMasterUrl() {
return kubernetesMasterUrl;
}

String getKubernetesApiToken() {
return kubernetesApiToken;
}

String getKubernetesCaCertificate() {
return kubernetesCaCertificate;
}
Expand All @@ -377,6 +373,10 @@ int getServicePort() {
return servicePort;
}

KubernetesTokenProvider getTokenProvider() {
return tokenProvider;
}

@Override
public String toString() {
return "Kubernetes Discovery properties: { "
Expand Down
22 changes: 22 additions & 0 deletions hazelcast/src/main/java/com/hazelcast/kubernetes/KubernetesTokenProvider.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
/*
* Copyright (c) 2008-2022, Hazelcast, Inc. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.hazelcast.kubernetes;

interface KubernetesTokenProvider {
String getToken();
}

30 changes: 30 additions & 0 deletions hazelcast/src/main/java/com/hazelcast/kubernetes/StaticTokenProvider.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
/*
* Copyright (c) 2008-2022, Hazelcast, Inc. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.hazelcast.kubernetes;

class StaticTokenProvider implements KubernetesTokenProvider {
private final String token;

StaticTokenProvider(String token) {
this.token = token;
}

@Override
public String getToken() {
return token;
}
}
62 changes: 59 additions & 3 deletions hazelcast/src/test/java/com/hazelcast/kubernetes/KubernetesClientTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,13 @@
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;

import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
Expand All @@ -45,11 +50,11 @@
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.instanceOf;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

public class KubernetesClientTest {
private static final String KUBERNETES_MASTER_IP = "localhost";

private static final String TOKEN = "sample-token";
private static final String CA_CERTIFICATE = "sample-ca-certificate";
private static final String NAMESPACE = "sample-namespace";
Expand All @@ -58,6 +63,9 @@ public class KubernetesClientTest {
@Rule
public WireMockRule wireMockRule = new WireMockRule(wireMockConfig().dynamicPort());

@Rule
public TemporaryFolder testFolder = new TemporaryFolder();

private KubernetesClient kubernetesClient;

@Before
Expand Down Expand Up @@ -1010,6 +1018,50 @@ public void endpointsWithoutNodeName() {
assertThat(formatPublic(result), containsInAnyOrder(ready("35.232.226.200", 32124), ready("35.232.226.201", 32124)));
}

@Test
public void apiAccessWithTokenRefresh() throws IOException {
// Token is read from the file, first token value is value-1
File file = testFolder.newFile("token");
Files.write(file.toPath(), "value-1".getBytes(StandardCharsets.UTF_8), StandardOpenOption.TRUNCATE_EXISTING);

stubFor(get(urlMatching("/apis/.*")).atPriority(1)
.withHeader("Authorization", equalTo("Bearer value-1"))
.willReturn(aResponse().withStatus(200).withBody("{}")));
stubFor(get(urlMatching("/api/.*")).atPriority(1)
.withHeader("Authorization", equalTo("Bearer value-1"))
.willReturn(aResponse().withStatus(200).withBody("{}")));
stubFor(get(urlMatching("/api/.*")).atPriority(1)
.withHeader("Authorization", equalTo("Bearer value-2"))
.willReturn(aResponse().withStatus(402).withBody("{}")));
stubFor(get(urlMatching("/apis/.*")).atPriority(1)
.withHeader("Authorization", equalTo("Bearer value-2"))
.willReturn(aResponse().withStatus(402).withBody("{}")));

KubernetesClient client = newKubernetesClient(new FileReaderTokenProvider(file.toString()));
client.endpoints();
assertFalse(client.isKnownExceptionAlreadyLogged());

// Token is rotated
Files.write(file.toPath(), "value-2".getBytes(StandardCharsets.UTF_8), StandardOpenOption.TRUNCATE_EXISTING);

// Api server will not accept token with old value
stubFor(get(urlMatching("/apis/.*")).atPriority(1)
.withHeader("Authorization", equalTo("Bearer value-1"))
.willReturn(aResponse().withStatus(402).withBody("{}")));
stubFor(get(urlMatching("/api/.*")).atPriority(1)
.withHeader("Authorization", equalTo("Bearer value-1"))
.willReturn(aResponse().withStatus(402).withBody("{}")));
stubFor(get(urlMatching("/api/.*")).atPriority(1)
.withHeader("Authorization", equalTo("Bearer value-2"))
.willReturn(aResponse().withStatus(200).withBody("{}")));
stubFor(get(urlMatching("/apis/.*")).atPriority(1)
.withHeader("Authorization", equalTo("Bearer value-2"))
.willReturn(aResponse().withStatus(200).withBody("{}")));

client.endpoints();
assertFalse(client.isKnownExceptionAlreadyLogged());
}

private static String serviceResponse(String ip) {
//language=JSON
return "{\n"
Expand Down Expand Up @@ -1443,12 +1495,16 @@ private KubernetesClient newKubernetesClient() {
return newKubernetesClient(false);
}

private KubernetesClient newKubernetesClient(KubernetesTokenProvider tokenProvider) {
String kubernetesMasterUrl = String.format("http://%s:%d", KUBERNETES_MASTER_IP, wireMockRule.port());
return new KubernetesClient(NAMESPACE, kubernetesMasterUrl, tokenProvider, CA_CERTIFICATE, RETRIES, ExposeExternallyMode.AUTO, true, null, null);
}

private KubernetesClient newKubernetesClient(boolean useNodeNameAsExternalAddress) {
return newKubernetesClient(useNodeNameAsExternalAddress, null, null);
}

private KubernetesClient newKubernetesClient(boolean useNodeNameAsExternalAddress, String servicePerPodLabelName, String servicePerPodLabelValue) {
String kubernetesMasterUrl = String.format("http://%s:%d", KUBERNETES_MASTER_IP, wireMockRule.port());
return newKubernetesClient(ExposeExternallyMode.AUTO, useNodeNameAsExternalAddress, servicePerPodLabelName, servicePerPodLabelValue);
}

Expand All @@ -1458,7 +1514,7 @@ private KubernetesClient newKubernetesClient(ExposeExternallyMode exposeExternal

private KubernetesClient newKubernetesClient(ExposeExternallyMode exposeExternally, boolean useNodeNameAsExternalAddress, String servicePerPodLabelName, String servicePerPodLabelValue, KubernetesApiProvider urlProvider) {
String kubernetesMasterUrl = String.format("http://%s:%d", KUBERNETES_MASTER_IP, wireMockRule.port());
return new KubernetesClient(NAMESPACE, kubernetesMasterUrl, TOKEN, CA_CERTIFICATE, RETRIES, exposeExternally, useNodeNameAsExternalAddress, servicePerPodLabelName, servicePerPodLabelValue, urlProvider);
return new KubernetesClient(NAMESPACE, kubernetesMasterUrl, new StaticTokenProvider(TOKEN), CA_CERTIFICATE, RETRIES, exposeExternally, useNodeNameAsExternalAddress, servicePerPodLabelName, servicePerPodLabelValue, urlProvider);
}

private static List<String> format(List<Endpoint> addresses) {
Expand Down
Loading

0 comments on commit 9a682d4

Please sign in to comment.