Skip to content

Commit

Permalink
RANGER-4914: Tagsync support for Ozone OFS paths
Browse files Browse the repository at this point in the history
  • Loading branch information
@fateh288
fateh288 committed Sep 11, 2024
1 parent 6e94858 commit 1debaa5
Show file tree
Hide file tree
Showing 2 changed files with 226 additions and 4 deletions.
92 changes: 90 additions & 2 deletions tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasOzoneResourceMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;

public class AtlasOzoneResourceMapper extends AtlasResourceMapper {
private static final Logger LOG = LoggerFactory.getLogger(AtlasOzoneResourceMapper.class);
Expand All @@ -51,9 +52,32 @@ public class AtlasOzoneResourceMapper extends AtlasResourceMapper {
private static final int IDX_CLUSTER_NAME = 3;
private static final int RESOURCE_COUNT = 4;

// This flag results in ofs atlas qualifiedName to parse paths similar to o3fs
public static final String PROP_LEGACY_PARSING = "ranger.tagsync.atlas.ozone.legacy.parsing.enabled";

public static final String PROP_OFS_KEY_DELIMITER = "ranger.tagsync.atlas.ozone.ofs.key_entity.separator";
public static final String PROP_OFS_BUCKET_DELIMITER = "ranger.tagsync.atlas.ozone.ofs.bucket_entity.separator";

private String ofsKeyDelimiter = "/";
private String ofsBucketDelimiter = "\\.";
private boolean legacyParsingEnabled;

public AtlasOzoneResourceMapper() {
super("ozone", SUPPORTED_ENTITY_TYPES);
}
@Override
public void initialize(Properties properties) {
super.initialize(properties);
this.legacyParsingEnabled = properties != null && Boolean.parseBoolean((String) this.properties.getOrDefault(
PROP_LEGACY_PARSING, "false"));
this.ofsKeyDelimiter = properties != null ?
(String) this.properties.getOrDefault(PROP_OFS_KEY_DELIMITER, this.ofsKeyDelimiter) : this.ofsKeyDelimiter;
this.ofsBucketDelimiter = properties != null ?
(String) this.properties.getOrDefault(PROP_OFS_BUCKET_DELIMITER, this.ofsBucketDelimiter) : this.ofsBucketDelimiter;
LOG.info("ofsKeyDelimiter="+this.ofsKeyDelimiter);
LOG.info("ofsBucketDelimiter="+this.ofsBucketDelimiter);
LOG.info(PROP_LEGACY_PARSING +"="+this.legacyParsingEnabled);
}

@Override
public RangerServiceResource buildResource(final RangerAtlasEntity entity) throws Exception {
Expand Down Expand Up @@ -134,10 +158,71 @@ public RangerServiceResource buildResource(final RangerAtlasEntity entity) throw
* o3fs://<volume name>@cm (ozone_key)
* o3fs://<volume name>.<bucket name>@<clusterName> (ozone_bucket)
* o3fs://<bucket name>.<volume name>.<ozone service id>/<key path>@<clusterName> (ozone_key)
* ofs://myvolume@cl1
* ofs://myvolume.mybucket@cl1
* ofs://ozone1/myvolume/mybucket/key1@cl1
* ofs://ozone1/myvolume/mybucket/mykey/key1/@cl1
*/
private String[] parseQualifiedName(String qualifiedName, String entityType) {
int idxProtocolSep = qualifiedName.indexOf(SEP_PROTOCOL);
String prefix = qualifiedName.substring(0,idxProtocolSep);
if (LOG.isDebugEnabled()){
LOG.debug("Prefix for qualifiedName="+qualifiedName+" is "+prefix);
}
if (this.legacyParsingEnabled){
return parseQualifiedNameO3FS(qualifiedName, entityType);
}
if (prefix.equals("ofs")) {
return parseQualifiedNameOFS(qualifiedName, entityType);
}
return parseQualifiedNameO3FS(qualifiedName, entityType);
}
private String[] parseQualifiedNameOFS(String qualifiedName, String entityType){
if(LOG.isDebugEnabled()){
LOG.debug("==>parseQualifiedNameOFS()");
}
String[] ret = new String[RESOURCE_COUNT];
if(StringUtils.isNotBlank(qualifiedName)) {
int idxClusterNameSep = qualifiedName.lastIndexOf(CLUSTER_DELIMITER);
if (idxClusterNameSep != -1) {
ret[IDX_CLUSTER_NAME] = qualifiedName.substring(idxClusterNameSep + CLUSTER_DELIMITER.length());
int idxProtocolSep = qualifiedName.indexOf(SEP_PROTOCOL);
if (idxProtocolSep != -1) {
int idxResourceStart = idxProtocolSep + SEP_PROTOCOL.length();
//Volume and bucket name have a "." as delimiter similar to o3fs.
if (StringUtils.equals(entityType, ENTITY_TYPE_OZONE_VOLUME)) {
// no problem in handling "." in volume name if volume is being tagged
ret[IDX_VOLUME] = qualifiedName.substring(idxResourceStart, idxClusterNameSep);
} else if (StringUtils.equals(entityType, ENTITY_TYPE_OZONE_BUCKET)) {
//anything before first "." is volume name, after that is bucket name. So, "." in volume name is invalid when tagging buckets
String[] resources = qualifiedName.substring(idxResourceStart, idxClusterNameSep).split(this.ofsBucketDelimiter,2);
ret[IDX_VOLUME] = resources.length > 0 ? resources[0] : null;
ret[IDX_BUCKET] = resources.length > 1 ? resources[1] : null;
} else if (StringUtils.equals(entityType, ENTITY_TYPE_OZONE_KEY)) {
// This is a special case wherein the delimiter is a "/" instead of a "." in the qualifiedName in ofs path
int ozoneServiceIdEnd = qualifiedName.indexOf(this.ofsKeyDelimiter, idxProtocolSep+SEP_PROTOCOL.length());
idxResourceStart = ozoneServiceIdEnd + 1;
String resourceString = qualifiedName.substring(idxResourceStart, idxClusterNameSep);
String[] resources = resourceString.split(this.ofsKeyDelimiter);
ret[IDX_VOLUME] = resources.length > 0 ? resources[0] : null;
ret[IDX_BUCKET] = resources.length > 1 ? resources[1] : null;
int keyStartIndex = resources[0].length() + resources[1].length() + 2*this.ofsKeyDelimiter.length();
ret[IDX_KEY] = resourceString.substring(keyStartIndex);
}
}
}
}
if(LOG.isDebugEnabled()){
LOG.debug("<==parseQualifiedNameOFS(qualifiedName="+qualifiedName+" entityType="+entityType+") volume="
+ret[IDX_VOLUME]+", bucket="+ret[IDX_BUCKET]+", key="+ret[IDX_KEY]+" clusterName="+ret[IDX_CLUSTER_NAME]);
}
return ret;
}
private String[] parseQualifiedNameO3FS(String qualifiedName, String entityType){
if(LOG.isDebugEnabled()){
LOG.debug("==>parseQualifiedNameO3FS()");
}
String[] ret = new String[RESOURCE_COUNT];

if(StringUtils.isNotBlank(qualifiedName)) {
int idxClusterNameSep = qualifiedName.lastIndexOf(CLUSTER_DELIMITER);

Expand Down Expand Up @@ -166,7 +251,10 @@ private String[] parseQualifiedName(String qualifiedName, String entityType) {
}
}
}

if(LOG.isDebugEnabled()){
LOG.debug("<==parseQualifiedNameO3FS(qualifiedName="+qualifiedName+" entityType="+entityType+") volume="
+ret[IDX_VOLUME]+", bucket="+ret[IDX_BUCKET]+", key="+ret[IDX_KEY]+" clusterName="+ret[IDX_CLUSTER_NAME]);
}
return ret;
}
}
138 changes: 136 additions & 2 deletions tagsync/src/test/java/org/apache/ranger/tagsync/process/TestOzoneResourceMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,11 @@
import org.apache.ranger.tagsync.source.atlas.AtlasOzoneResourceMapper;
import org.apache.ranger.tagsync.source.atlasrest.RangerAtlasEntity;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;

import java.util.Collections;
import java.util.Properties;

import static org.apache.ranger.tagsync.source.atlas.AtlasOzoneResourceMapper.*;
import static org.apache.ranger.tagsync.source.atlas.AtlasResourceMapper.ENTITY_ATTRIBUTE_QUALIFIED_NAME;
Expand All @@ -37,13 +39,24 @@ public class TestOzoneResourceMapper {
private static final String KEY_QUALIFIED_NAME = "o3fs://mybucket.myvolume.ozone1/mykey.txt@cl1" ;
private static final String KEY_PATH_QUALIFIED_NAME = "o3fs://mybucket.myvolume.ozone1/mykey/key1/@cl1";

private static final String VOLUME_QUALIFIED_NAME_OFS = "ofs://myvolume@cl1";
private static final String BUCKET_QUALIFIED_NAME_OFS = "ofs://myvolume.mybucket@cl1";
private static final String KEY_QUALIFIED_NAME_OFS = "ofs://ozone1/myvolume/mybucket/mykey.txt@cl1";
private static final String KEY_PATH_QUALIFIED_NAME_OFS = "ofs://ozone1/myvolume/mybucket/mykey/key1/@cl1";

private static final String SERVICE_NAME = "cl1_ozone";
private static final String VOLUME_NAME = "myvolume";
private static final String BUCKET_NAME = "mybucket";
private static final String KEY_NAME = "mykey.txt";
private static final String KEY_PATH = "mykey/key1/";

AtlasOzoneResourceMapper resourceMapper = new AtlasOzoneResourceMapper();

static AtlasOzoneResourceMapper resourceMapper = new AtlasOzoneResourceMapper();

@BeforeClass
public static void init(){
resourceMapper.initialize(new Properties());
}

@Test
public void testVolumeEntity() throws Exception {
Expand All @@ -54,7 +67,15 @@ public void testVolumeEntity() throws Exception {
assertResourceElementCount(resource, 1);
assertResourceElementValue(resource, RANGER_TYPE_OZONE_VOLUME, VOLUME_NAME);
}
@Test
public void testVolumeEntityOFS() throws Exception {
RangerAtlasEntity entity = getEntity(ENTITY_TYPE_OZONE_VOLUME, VOLUME_QUALIFIED_NAME_OFS);
RangerServiceResource resource = resourceMapper.buildResource(entity);

Assert.assertEquals(SERVICE_NAME, resource.getServiceName());
assertResourceElementCount(resource, 1);
assertResourceElementValue(resource, RANGER_TYPE_OZONE_VOLUME, VOLUME_NAME);
}
@Test
public void testBucketEntity() throws Exception {
RangerAtlasEntity entity = getEntity(ENTITY_TYPE_OZONE_BUCKET, BUCKET_QUALIFIED_NAME);
Expand All @@ -65,7 +86,16 @@ public void testBucketEntity() throws Exception {
assertResourceElementValue(resource, RANGER_TYPE_OZONE_VOLUME, VOLUME_NAME);
assertResourceElementValue(resource, RANGER_TYPE_OZONE_BUCKET, BUCKET_NAME);
}
@Test
public void testBucketEntityOFS() throws Exception {
RangerAtlasEntity entity = getEntity(ENTITY_TYPE_OZONE_BUCKET, BUCKET_QUALIFIED_NAME_OFS);
RangerServiceResource resource = resourceMapper.buildResource(entity);

Assert.assertEquals(SERVICE_NAME, resource.getServiceName());
assertResourceElementCount(resource, 2);
assertResourceElementValue(resource, RANGER_TYPE_OZONE_VOLUME, VOLUME_NAME);
assertResourceElementValue(resource, RANGER_TYPE_OZONE_BUCKET, BUCKET_NAME);
}
@Test
public void testKeyEntity() throws Exception {
RangerAtlasEntity entity = getEntity(ENTITY_TYPE_OZONE_KEY, KEY_QUALIFIED_NAME);
Expand All @@ -77,7 +107,17 @@ public void testKeyEntity() throws Exception {
assertResourceElementValue(resource, RANGER_TYPE_OZONE_BUCKET, BUCKET_NAME);
assertResourceElementValue(resource, RANGER_TYPE_OZONE_KEY, KEY_NAME);
}
@Test
public void testKeyEntityOFS() throws Exception {
RangerAtlasEntity entity = getEntity(ENTITY_TYPE_OZONE_KEY, KEY_QUALIFIED_NAME_OFS);
RangerServiceResource resource = resourceMapper.buildResource(entity);

Assert.assertEquals(SERVICE_NAME, resource.getServiceName());
assertResourceElementCount(resource, 3);
assertResourceElementValue(resource, RANGER_TYPE_OZONE_VOLUME, VOLUME_NAME);
assertResourceElementValue(resource, RANGER_TYPE_OZONE_BUCKET, BUCKET_NAME);
assertResourceElementValue(resource, RANGER_TYPE_OZONE_KEY, KEY_NAME);
}
@Test
public void testKey2Entity() throws Exception {
RangerAtlasEntity entity = getEntity(ENTITY_TYPE_OZONE_KEY, KEY_PATH_QUALIFIED_NAME);
Expand All @@ -89,7 +129,101 @@ public void testKey2Entity() throws Exception {
assertResourceElementValue(resource, RANGER_TYPE_OZONE_BUCKET, BUCKET_NAME);
assertResourceElementValue(resource, RANGER_TYPE_OZONE_KEY, KEY_PATH);
}

@Test
public void testKey2EntityOFS() throws Exception {
RangerAtlasEntity entity = getEntity(ENTITY_TYPE_OZONE_KEY, KEY_PATH_QUALIFIED_NAME_OFS);
RangerServiceResource resource = resourceMapper.buildResource(entity);
Assert.assertEquals(SERVICE_NAME, resource.getServiceName());
assertResourceElementCount(resource, 3);
assertResourceElementValue(resource, RANGER_TYPE_OZONE_VOLUME, VOLUME_NAME);
assertResourceElementValue(resource, RANGER_TYPE_OZONE_BUCKET, BUCKET_NAME);
assertResourceElementValue(resource, RANGER_TYPE_OZONE_KEY, KEY_PATH);
}
@Test
public void testKeyEntityOFSLegacyDotDelimiter() throws Exception {
AtlasOzoneResourceMapper legacyResourceMapper = new AtlasOzoneResourceMapper();
Properties legacyProperties = new Properties();
legacyProperties.setProperty(PROP_LEGACY_PARSING, "true");
legacyResourceMapper.initialize(legacyProperties);
String qualifiedName = "ofs://mybucket.myvolume.ozone1/mykey.txt@cl1";
RangerAtlasEntity entity = getEntity(ENTITY_TYPE_OZONE_KEY, qualifiedName);
RangerServiceResource resource = legacyResourceMapper.buildResource(entity);
Assert.assertEquals(SERVICE_NAME, resource.getServiceName());
assertResourceElementCount(resource, 3);
assertResourceElementValue(resource, RANGER_TYPE_OZONE_VOLUME, VOLUME_NAME);
assertResourceElementValue(resource, RANGER_TYPE_OZONE_BUCKET, BUCKET_NAME);
assertResourceElementValue(resource, RANGER_TYPE_OZONE_KEY, KEY_NAME);
}
@Test
public void testInvalidKeyEntityOFSLegacyDotDelimiter() throws Exception {
AtlasOzoneResourceMapper legacyResourceMapper = new AtlasOzoneResourceMapper();
Properties legacyProperties = new Properties();
legacyProperties.setProperty(PROP_LEGACY_PARSING, "true");
legacyResourceMapper.initialize(legacyProperties);
RangerAtlasEntity entity = getEntity(ENTITY_TYPE_OZONE_KEY, KEY_PATH_QUALIFIED_NAME_OFS);
try {
RangerServiceResource resource = legacyResourceMapper.buildResource(entity);
Assert.assertFalse("Expected buildResource() to fail. But it returned " + resource+". "
+ "'/' not supported as delimiter when legacy flag is enabled", true);
} catch (Exception excp) {
System.out.println("Exception was as expected: "+ KEY_PATH_QUALIFIED_NAME_OFS+
" cannot be parsed when property"+ PROP_LEGACY_PARSING +" is true");
}
}
@Test
public void testVolumeEntityWithDotOFS() throws Exception {
String qualifiedName = "ofs://myvolume.volpostfix@cl1";
String expectedVolumeName = "myvolume.volpostfix";
RangerAtlasEntity entity = getEntity(ENTITY_TYPE_OZONE_VOLUME, qualifiedName);
RangerServiceResource resource = resourceMapper.buildResource(entity);
Assert.assertEquals(SERVICE_NAME, resource.getServiceName());
assertResourceElementCount(resource, 1);
assertResourceElementValue(resource, RANGER_TYPE_OZONE_VOLUME, expectedVolumeName);
}
@Test
public void testBucketEntityWithDotOFS() throws Exception {
String qualifiedName = "ofs://myvolume.bucketprefix.mybucket.bucketpostfix@cl1";
String expectedVolumeName = "myvolume";
String expectedBucketName = "bucketprefix.mybucket.bucketpostfix";
RangerAtlasEntity entity = getEntity(ENTITY_TYPE_OZONE_BUCKET, qualifiedName);
RangerServiceResource resource = resourceMapper.buildResource(entity);
Assert.assertEquals(SERVICE_NAME, resource.getServiceName());
assertResourceElementCount(resource, 2);
assertResourceElementValue(resource, RANGER_TYPE_OZONE_VOLUME, expectedVolumeName);
assertResourceElementValue(resource, RANGER_TYPE_OZONE_BUCKET, expectedBucketName);
}
@Test
public void testKeyEntityWithDotOFS() throws Exception {
String qualifiedName = "ofs://ozone1/myvolume.volumepostfix/mybucket.bucketpostfix/keypath/keyprefix.mykey.txt@cl1";
String expectedVolumeName = "myvolume.volumepostfix";
String expectedBucketName = "mybucket.bucketpostfix";
String expectedKeyName = "keypath/keyprefix.mykey.txt";
RangerAtlasEntity entity = getEntity(ENTITY_TYPE_OZONE_KEY, qualifiedName);
RangerServiceResource resource = resourceMapper.buildResource(entity);
Assert.assertEquals(SERVICE_NAME, resource.getServiceName());
assertResourceElementCount(resource, 3);
assertResourceElementValue(resource, RANGER_TYPE_OZONE_VOLUME, expectedVolumeName);
assertResourceElementValue(resource, RANGER_TYPE_OZONE_BUCKET, expectedBucketName);
assertResourceElementValue(resource, RANGER_TYPE_OZONE_KEY, expectedKeyName);
}
@Test
public void testBucketEntityWithSlashOFS() throws Exception {
//future work : scenario when atlas fixes the bucket delimiter from "." to "/";
AtlasOzoneResourceMapper afterDelimiterFixResourceMapper = new AtlasOzoneResourceMapper();
Properties properties = new Properties();
properties.setProperty(PROP_OFS_BUCKET_DELIMITER, "/");
afterDelimiterFixResourceMapper.initialize(properties);
//both volume and bucket name could have a "." in it after this fix
String qualifiedName = "ofs://myvolume.volumepostfix/mybucket.bucketpostfix@cl1";
String expectedVolumeName = "myvolume.volumepostfix";
String expectedBucketName = "mybucket.bucketpostfix";
RangerAtlasEntity entity = getEntity(ENTITY_TYPE_OZONE_BUCKET, qualifiedName);
RangerServiceResource resource = afterDelimiterFixResourceMapper.buildResource(entity);
Assert.assertEquals(SERVICE_NAME, resource.getServiceName());
assertResourceElementCount(resource, 2);
assertResourceElementValue(resource, RANGER_TYPE_OZONE_VOLUME, expectedVolumeName);
assertResourceElementValue(resource, RANGER_TYPE_OZONE_BUCKET, expectedBucketName);
}
@Test
public void testInvalidEntityType() {
assertException(getEntity("Unknown", KEY_PATH_QUALIFIED_NAME), "unrecognized entity-type");
Expand Down

0 comments on commit 1debaa5

Please sign in to comment.