Skip to content

Commit

Permalink
Remove zookeeper part 1
Browse files Browse the repository at this point in the history
  • Loading branch information
@yuzhu
yuzhu committed Aug 21, 2023
1 parent 46d0280 commit 58e5b26
Show file tree
Hide file tree
Showing 44 changed files with 54 additions and 3,462 deletions.
23 changes: 0 additions & 23 deletions dora/core/client/fs/src/test/java/alluxio/client/file/FileSystemFactoryTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
import alluxio.master.MasterInquireClient;
import alluxio.security.User;
import alluxio.uri.MultiMasterAuthority;
import alluxio.uri.ZookeeperAuthority;

import org.junit.After;
import org.junit.Before;
Expand All @@ -37,9 +36,7 @@
import java.io.Closeable;
import java.io.IOException;
import java.security.Principal;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import javax.security.auth.Subject;

Expand Down Expand Up @@ -88,26 +85,6 @@ public void multiMasterFileSystemCacheTest() {
}
}

@Test
public void zkFileSystemCacheTest() {
Map<String, String> sysProps = new HashMap<>();
sysProps.put(PropertyKey.ZOOKEEPER_ENABLED.getName(), Boolean.toString(true));
sysProps.put(PropertyKey.ZOOKEEPER_ADDRESS.getName(), "zk@192.168.0.5");
sysProps.put(PropertyKey.ZOOKEEPER_ELECTION_PATH.getName(), "/alluxio/leader");

try (Closeable p = new SystemPropertyRule(sysProps).toResource()) {
Configuration.reloadProperties();
AlluxioConfiguration conf = Configuration.global();
MasterInquireClient.ConnectDetails connectDetails =
MasterInquireClient.Factory.getConnectDetails(conf);
// Make sure we have a Zookeeper authority
assertTrue(connectDetails.toAuthority() instanceof ZookeeperAuthority);
fileSystemCacheTest();
} catch (IOException e) {
fail("Unable to set system properties");
}
}

@Test
public void nullSubjectTest() {
assertThrows(NullPointerException.class, () -> FileSystem.Factory.get(null));
Expand Down
37 changes: 2 additions & 35 deletions dora/core/client/hdfs/src/main/java/alluxio/hadoop/FileSystem.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,6 @@
import alluxio.uri.MultiMasterAuthority;
import alluxio.uri.SingleMasterAuthority;
import alluxio.uri.UnknownAuthority;
import alluxio.uri.ZookeeperAuthority;
import alluxio.uri.ZookeeperLogicalAuthority;

import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
Expand Down Expand Up @@ -66,25 +64,18 @@ public String getScheme() {

@Override
protected boolean isZookeeperMode() {
return mFileSystem.getConf().getBoolean(PropertyKey.ZOOKEEPER_ENABLED);
return false;
}

@Override
protected Map<String, Object> getConfigurationFromUri(URI uri, Configuration conf) {
AlluxioURI alluxioUri = new AlluxioURI(uri.toString());
Map<String, Object> alluxioConfProperties = new HashMap<>();

if (alluxioUri.getAuthority() instanceof ZookeeperAuthority) {
ZookeeperAuthority authority = (ZookeeperAuthority) alluxioUri.getAuthority();
alluxioConfProperties.put(PropertyKey.ZOOKEEPER_ENABLED.getName(), true);
alluxioConfProperties.put(PropertyKey.ZOOKEEPER_ADDRESS.getName(),
authority.getZookeeperAddress());
} else if (alluxioUri.getAuthority() instanceof SingleMasterAuthority) {
if (alluxioUri.getAuthority() instanceof SingleMasterAuthority) {
SingleMasterAuthority authority = (SingleMasterAuthority) alluxioUri.getAuthority();
alluxioConfProperties.put(PropertyKey.MASTER_HOSTNAME.getName(), authority.getHost());
alluxioConfProperties.put(PropertyKey.MASTER_RPC_PORT.getName(), authority.getPort());
alluxioConfProperties.put(PropertyKey.ZOOKEEPER_ENABLED.getName(), false);
alluxioConfProperties.put(PropertyKey.ZOOKEEPER_ADDRESS.getName(), null);
// Unset the embedded journal related configuration
// to support alluxio URI has the highest priority
alluxioConfProperties.put(PropertyKey.MASTER_EMBEDDED_JOURNAL_ADDRESSES.getName(), null);
Expand All @@ -93,9 +84,6 @@ protected Map<String, Object> getConfigurationFromUri(URI uri, Configuration con
MultiMasterAuthority authority = (MultiMasterAuthority) alluxioUri.getAuthority();
alluxioConfProperties.put(PropertyKey.MASTER_RPC_ADDRESSES.getName(),
authority.getMasterAddresses());
// Unset the zookeeper configuration to support alluxio URI has the highest priority
alluxioConfProperties.put(PropertyKey.ZOOKEEPER_ENABLED.getName(), false);
alluxioConfProperties.put(PropertyKey.ZOOKEEPER_ADDRESS.getName(), null);
} else if (alluxioUri.getAuthority() instanceof EmbeddedLogicalAuthority) {
EmbeddedLogicalAuthority authority = (EmbeddedLogicalAuthority) alluxioUri.getAuthority();
String masterNamesConfKey = PropertyKey.Template.MASTER_LOGICAL_NAMESERVICES
Expand All @@ -115,27 +103,6 @@ protected Map<String, Object> getConfigurationFromUri(URI uri, Configuration con

alluxioConfProperties.put(PropertyKey.MASTER_RPC_ADDRESSES.getName(),
masterRpcAddress.toString());
alluxioConfProperties.put(PropertyKey.ZOOKEEPER_ENABLED.getName(), false);
alluxioConfProperties.put(PropertyKey.ZOOKEEPER_ADDRESS.getName(), null);
} else if (alluxioUri.getAuthority() instanceof ZookeeperLogicalAuthority) {
ZookeeperLogicalAuthority authority = (ZookeeperLogicalAuthority) alluxioUri.getAuthority();
String zkNodesConfKey = PropertyKey.Template.MASTER_LOGICAL_ZOOKEEPER_NAMESERVICES
.format(authority.getLogicalName()).getName();
String[] zkNodeNames = conf.getTrimmedStrings(zkNodesConfKey);
Preconditions.checkArgument(zkNodeNames.length != 0,
"Invalid uri. You must set %s to use the logical name", zkNodesConfKey);

StringJoiner zkAddress = new StringJoiner(",");
for (String zkName : zkNodeNames) {
String name = PropertyKey.Template.MASTER_LOGICAL_ZOOKEEPER_ADDRESS
.format(authority.getLogicalName(), zkName).getName();
String address = conf.get(name);
Preconditions.checkArgument(address != null, "You need to set %s", name);
zkAddress.add(address);
}

alluxioConfProperties.put(PropertyKey.ZOOKEEPER_ENABLED.getName(), true);
alluxioConfProperties.put(PropertyKey.ZOOKEEPER_ADDRESS.getName(), zkAddress.toString());
}
return alluxioConfProperties;
}
Expand Down
15 changes: 0 additions & 15 deletions dora/core/client/hdfs/src/test/java/alluxio/hadoop/AbstractFileSystemApiTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,21 +75,6 @@ public void validAuthorityNoWarning() throws IOException {
assertFalse(loggedAuthorityWarning());
}

@Test
@DoraTestTodoItem(action = DoraTestTodoItem.Action.FIX, owner = "Jiacheng",
comment = "fix test because the URI is general")
@Ignore
public void parseZkUriWithPlusDelimiters() throws Exception {
org.apache.hadoop.fs.FileSystem fs = FileSystem.get(URI.create("alluxio://zk@a:0+b:1+c:2/"),
new org.apache.hadoop.conf.Configuration());
assertTrue(fs instanceof AbstractFileSystem);
AbstractFileSystem afs = (AbstractFileSystem) fs;
assertTrue(afs.mFileSystem.getConf()
.getBoolean(PropertyKey.ZOOKEEPER_ENABLED));
assertEquals("a:0,b:1,c:2", afs.mFileSystem.getConf()
.get(PropertyKey.ZOOKEEPER_ADDRESS));
}

private boolean loggedAuthorityWarning() {
return mTestLogger.wasLogged("Authority .* is unknown");
}
Expand Down
179 changes: 0 additions & 179 deletions dora/core/client/hdfs/src/test/java/alluxio/hadoop/AbstractFileSystemTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,85 +123,13 @@ public void after() {
HadoopClientTestUtils.disableMetrics(mConfiguration);
}

@Test
public void hadoopShouldLoadFileSystemWithSingleZkUri() throws Exception {
org.apache.hadoop.conf.Configuration conf = getConf();
URI uri = URI.create(Constants.HEADER + "zk@zkHost:2181/tmp/path.txt");

FileSystem hfs = getHadoopFilesystem(org.apache.hadoop.fs.FileSystem.get(uri, conf));
assertTrue(hfs.mFileSystem.getConf().getBoolean(PropertyKey.ZOOKEEPER_ENABLED));
assertEquals("zkHost:2181", hfs.mFileSystem.getConf().get(PropertyKey.ZOOKEEPER_ADDRESS));
}

@Test
public void hadoopShouldLoadFileSystemWithMultipleZkUri() throws Exception {
org.apache.hadoop.conf.Configuration conf = getConf();
URI uri = URI.create(Constants.HEADER + "zk@host1:2181,host2:2181,host3:2181/tmp/path.txt");
org.apache.hadoop.fs.FileSystem fs = org.apache.hadoop.fs.FileSystem.get(uri, conf);

FileSystem hfs = getHadoopFilesystem(fs);
assertTrue(hfs.mFileSystem.getConf().getBoolean(PropertyKey.ZOOKEEPER_ENABLED));
assertEquals("host1:2181,host2:2181,host3:2181",
hfs.mFileSystem.getConf().get(PropertyKey.ZOOKEEPER_ADDRESS));

uri = URI.create(Constants.HEADER + "zk@host1:2181;host2:2181;host3:2181/tmp/path.txt");
fs = getHadoopFilesystem(org.apache.hadoop.fs.FileSystem.get(uri, conf));
assertTrue(hfs.mFileSystem.getConf().getBoolean(PropertyKey.ZOOKEEPER_ENABLED));
assertEquals("host1:2181,host2:2181,host3:2181",
hfs.mFileSystem.getConf().get(PropertyKey.ZOOKEEPER_ADDRESS));
}

@Test
public void fsShouldSetPropertyConfWithLogicalUriConfig() throws Exception {
URI uri = URI.create("alluxio://ebj@logical/path");
Configuration conf = getConf();
conf.set(PropertyKey.Template.MASTER_LOGICAL_NAMESERVICES.format("logical").getName(),
"master1,master2,master3");
conf.set(
PropertyKey.Template.MASTER_LOGICAL_RPC_ADDRESS.format("logical", "master1").getName(),
"host1:19998");
conf.set(
PropertyKey.Template.MASTER_LOGICAL_RPC_ADDRESS.format("logical", "master2").getName(),
"host2:19998");
conf.set(
PropertyKey.Template.MASTER_LOGICAL_RPC_ADDRESS.format("logical", "master3").getName(),
"host3:19998");
AbstractFileSystem afs = new alluxio.hadoop.FileSystem();
afs.initialize(uri, conf);
assertEquals("host1:19998,host2:19998,host3:19998",
afs.mFileSystem.getConf().get(PropertyKey.MASTER_RPC_ADDRESSES));
assertFalse(afs.mFileSystem.getConf().getBoolean(PropertyKey.ZOOKEEPER_ENABLED));
}

@Test
public void fsShouldTriggersExceptionWithUnknownLogicalUriWith() throws Exception {
URI uri = URI.create("alluxio://ebj@logical/path");
AbstractFileSystem afs = new alluxio.hadoop.FileSystem();
assertThrows(Exception.class, () -> afs.initialize(uri, getConf()));
}

@Test
public void fsShouldSetPropertyConfWithZkLogicalUriConfig() throws Exception {
URI uri = URI.create("alluxio://zk@logical/path");
Configuration conf = getConf();
conf.set(PropertyKey.Template.MASTER_LOGICAL_ZOOKEEPER_NAMESERVICES.format("logical").getName(),
"node1,node2,node3");
conf.set(
PropertyKey.Template.MASTER_LOGICAL_ZOOKEEPER_ADDRESS.format("logical", "node1").getName(),
"host1:2181");
conf.set(
PropertyKey.Template.MASTER_LOGICAL_ZOOKEEPER_ADDRESS.format("logical", "node2").getName(),
"host2:2181");
conf.set(
PropertyKey.Template.MASTER_LOGICAL_ZOOKEEPER_ADDRESS.format("logical", "node3").getName(),
"host3:2181");
AbstractFileSystem afs = new alluxio.hadoop.FileSystem();
afs.initialize(uri, conf);
assertEquals("host1:2181,host2:2181,host3:2181",
afs.mFileSystem.getConf().get(PropertyKey.ZOOKEEPER_ADDRESS));
assertTrue(afs.mFileSystem.getConf().getBoolean(PropertyKey.ZOOKEEPER_ENABLED));
}

@Test
public void fsShouldTriggersExceptionWithUnknownZkLogicalUriWith() {
URI uri = URI.create("alluxio://zk@logical/path");
Expand All @@ -214,17 +142,13 @@ public void fsShouldSetPropertyConfWithMultiMasterUri() throws Exception {
URI uri = URI.create("alluxio://host1:19998,host2:19998,host3:19998/path");
AbstractFileSystem afs = new alluxio.hadoop.FileSystem();
afs.initialize(uri, getConf());
assertFalse(afs.mFileSystem.getConf()
.getBoolean(PropertyKey.ZOOKEEPER_ENABLED));
assertEquals("host1:19998,host2:19998,host3:19998",
afs.mFileSystem.getConf().get(PropertyKey.MASTER_RPC_ADDRESSES));

uri = URI.create("alluxio://host1:19998;host2:19998;host3:19998/path");
afs = new FileSystem();
afs.initialize(uri, getConf());

assertFalse(afs.mFileSystem.getConf()
.getBoolean(PropertyKey.ZOOKEEPER_ENABLED));
assertEquals("host1:19998,host2:19998,host3:19998",
afs.mFileSystem.getConf().get(PropertyKey.MASTER_RPC_ADDRESSES));
}
Expand All @@ -249,76 +173,18 @@ public void hadoopShouldLoadFileSystemWhenConfigured() throws Exception {
Map<PropertyKey, Object> properties = new HashMap<>();
properties.put(PropertyKey.MASTER_HOSTNAME, uri.getHost());
properties.put(PropertyKey.MASTER_RPC_PORT, uri.getPort());
properties.put(PropertyKey.ZOOKEEPER_ENABLED, false);
properties.put(PropertyKey.ZOOKEEPER_ADDRESS, null);
try (Closeable c = new ConfigurationRule(properties, mConfiguration).toResource()) {
final org.apache.hadoop.fs.FileSystem fs = org.apache.hadoop.fs.FileSystem.get(uri, conf);
assertTrue(fs instanceof FileSystem);
}
}

@Test
public void resetContextUsingZookeeperUris() throws Exception {
// Change to signle zookeeper uri
URI uri = URI.create(Constants.HEADER + "zk@zkHost:2181/");
FileSystem fs = getHadoopFilesystem(org.apache.hadoop.fs.FileSystem.get(uri, getConf()));

assertTrue(fs.mFileSystem.getConf().getBoolean(PropertyKey.ZOOKEEPER_ENABLED));
assertEquals("zkHost:2181", fs.mFileSystem.getConf().get(PropertyKey.ZOOKEEPER_ADDRESS));

uri = URI.create(Constants.HEADER + "zk@host1:2181,host2:2181,host3:2181/tmp/path.txt");
fs = getHadoopFilesystem(org.apache.hadoop.fs.FileSystem.get(uri, getConf()));

assertTrue(fs.mFileSystem.getConf().getBoolean(PropertyKey.ZOOKEEPER_ENABLED));
assertEquals("host1:2181,host2:2181,host3:2181",
fs.mFileSystem.getConf().get(PropertyKey.ZOOKEEPER_ADDRESS));

uri = URI.create(Constants.HEADER + "zk@host1:2181;host2:2181;host3:2181/tmp/path.txt");
fs = getHadoopFilesystem(org.apache.hadoop.fs.FileSystem.get(uri, getConf()));
assertTrue(fs.mFileSystem.getConf().getBoolean(PropertyKey.ZOOKEEPER_ENABLED));
assertEquals("host1:2181,host2:2181,host3:2181",
fs.mFileSystem.getConf().get(PropertyKey.ZOOKEEPER_ADDRESS));
}

@Test
public void resetContextFromZkUriToNonZkUri() throws Exception {
org.apache.hadoop.conf.Configuration conf = getConf();
URI uri = URI.create(Constants.HEADER + "zk@zkHost:2181/tmp/path.txt");
FileSystem fs = getHadoopFilesystem(org.apache.hadoop.fs.FileSystem.get(uri, conf));
assertTrue(fs.mFileSystem.getConf().getBoolean(PropertyKey.ZOOKEEPER_ENABLED));
assertEquals("zkHost:2181", fs.mFileSystem.getConf().get(PropertyKey.ZOOKEEPER_ADDRESS));

URI otherUri = URI.create(Constants.HEADER + "alluxioHost:19998/tmp/path.txt");
fs = getHadoopFilesystem(org.apache.hadoop.fs.FileSystem.get(otherUri, conf));
assertEquals("alluxioHost", fs.mFileSystem.getConf().get(PropertyKey.MASTER_HOSTNAME));
assertEquals(19998, fs.mFileSystem.getConf().get(PropertyKey.MASTER_RPC_PORT));
assertFalse(fs.mFileSystem.getConf().getBoolean(PropertyKey.ZOOKEEPER_ENABLED));
assertFalse(fs.mFileSystem.getConf().isSet(PropertyKey.ZOOKEEPER_ADDRESS));
}

@Test
public void resetContextUsingMultiMasterUris() throws Exception {
// Change to multi-master uri
URI uri = URI.create(Constants.HEADER + "host1:19998,host2:19998,host3:19998/tmp/path.txt");
FileSystem fs = getHadoopFilesystem(org.apache.hadoop.fs.FileSystem.get(uri, getConf()));
assertFalse(fs.mFileSystem.getConf().getBoolean(PropertyKey.ZOOKEEPER_ENABLED));
assertEquals("host1:19998,host2:19998,host3:19998",
fs.mFileSystem.getConf().get(PropertyKey.MASTER_RPC_ADDRESSES));
}

@Test
public void resetContextFromZookeeperToMultiMaster() throws Exception {
URI uri = URI.create(Constants.HEADER + "zk@zkHost:2181/tmp/path.txt");
FileSystem fs = getHadoopFilesystem(org.apache.hadoop.fs.FileSystem.get(uri, getConf()));
assertTrue(fs.mFileSystem.getConf().getBoolean(PropertyKey.ZOOKEEPER_ENABLED));
assertEquals("zkHost:2181", fs.mFileSystem.getConf().get(PropertyKey.ZOOKEEPER_ADDRESS));

uri = URI.create(Constants.HEADER + "host1:19998,host2:19998,host3:19998/tmp/path.txt");
fs = getHadoopFilesystem(org.apache.hadoop.fs.FileSystem.get(uri, getConf()));

assertFalse(fs.mFileSystem.getConf().getBoolean(PropertyKey.ZOOKEEPER_ENABLED));
assertEquals(3,
ConfigurationUtils.getMasterRpcAddresses(fs.mFileSystem.getConf()).size());
assertEquals("host1:19998,host2:19998,host3:19998",
fs.mFileSystem.getConf().get(PropertyKey.MASTER_RPC_ADDRESSES));
}
Expand All @@ -328,7 +194,6 @@ public void resetContextFromMultiMasterToSingleMaster() throws Exception {
URI uri = URI.create(Constants.HEADER + "host1:19998,host2:19998,host3:19998/tmp/path.txt");
FileSystem fs = getHadoopFilesystem(org.apache.hadoop.fs.FileSystem.get(uri, getConf()));

assertFalse(fs.mFileSystem.getConf().getBoolean(PropertyKey.ZOOKEEPER_ENABLED));
assertEquals(3,
ConfigurationUtils.getMasterRpcAddresses(fs.mFileSystem.getConf()).size());
assertEquals("host1:19998,host2:19998,host3:19998",
Expand All @@ -337,7 +202,6 @@ public void resetContextFromMultiMasterToSingleMaster() throws Exception {
uri = URI.create(Constants.HEADER + "host:19998/tmp/path.txt");
fs = getHadoopFilesystem(org.apache.hadoop.fs.FileSystem.get(uri, getConf()));

assertFalse(fs.mFileSystem.getConf().getBoolean(PropertyKey.ZOOKEEPER_ENABLED));
assertEquals(PropertyKey.MASTER_JOURNAL_TYPE.getDefaultValue(),
fs.mFileSystem.getConf().get(PropertyKey.MASTER_JOURNAL_TYPE));
assertEquals(1,
Expand Down Expand Up @@ -547,49 +411,6 @@ public void initializeWithFullPrincipalUgi() throws Exception {
// FileSystem.create would have thrown an exception if the initialization failed.
}

@Test
public void initializeWithZookeeperSystemProperties() throws Exception {
HashMap<String, String> sysProps = new HashMap<>();
sysProps.put(PropertyKey.ZOOKEEPER_ENABLED.getName(), "true");
sysProps.put(PropertyKey.ZOOKEEPER_ADDRESS.getName(), "zkHost:2181");
try (Closeable p = new SystemPropertyRule(sysProps).toResource()) {
alluxio.conf.Configuration.reloadProperties();
URI uri = URI.create("alluxio:///");
FileSystem fs = getHadoopFilesystem(org.apache.hadoop.fs.FileSystem.get(uri, getConf()));

assertTrue(fs.mFileSystem.getConf().getBoolean(PropertyKey.ZOOKEEPER_ENABLED));
assertEquals("zkHost:2181", fs.mFileSystem.getConf().get(PropertyKey.ZOOKEEPER_ADDRESS));
}
}

@Test
public void initializeWithZookeeperUriAndSystemProperty() throws Exception {
// When URI and system property both have Zookeeper configuration,
// those in the URI has the highest priority.
try (Closeable p = new SystemPropertyRule(
PropertyKey.ZOOKEEPER_ENABLED.getName(), "false").toResource()) {
alluxio.conf.Configuration.reloadProperties();
URI uri = URI.create("alluxio://zk@zkHost:2181");
FileSystem fs = getHadoopFilesystem(org.apache.hadoop.fs.FileSystem.get(uri, getConf()));
assertTrue(fs.mFileSystem.getConf().getBoolean(PropertyKey.ZOOKEEPER_ENABLED));
assertEquals("zkHost:2181", fs.mFileSystem.getConf().get(PropertyKey.ZOOKEEPER_ADDRESS));
fs.close();
}

HashMap<String, String> sysProps = new HashMap<>();
sysProps.put(PropertyKey.ZOOKEEPER_ENABLED.getName(), "true");
sysProps.put(PropertyKey.ZOOKEEPER_ADDRESS.getName(), "zkHost1:2181");
try (Closeable p = new SystemPropertyRule(sysProps).toResource()) {
alluxio.conf.Configuration.reloadProperties();
URI uri = URI.create("alluxio://zk@zkHost2:2181");
FileSystem fs = getHadoopFilesystem(org.apache.hadoop.fs.FileSystem.get(uri, getConf()));
assertTrue(fs.mFileSystem.getConf().getBoolean(PropertyKey.ZOOKEEPER_ENABLED));
assertEquals("zkHost2:2181", fs.mFileSystem.getConf().get(PropertyKey.ZOOKEEPER_ADDRESS));
fs.close();
}
alluxio.conf.Configuration.reloadProperties();
}

@Test
public void getBlockLocationsOnlyInAlluxio() throws Exception {
WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1").setDataPort(1234);
Expand Down
Loading

0 comments on commit 58e5b26

Please sign in to comment.