From 826654f944515d1e4c81545e47d1d28b1f0c953d Mon Sep 17 00:00:00 2001 From: Liam Huffman Date: Mon, 9 Dec 2024 14:45:52 -0800 Subject: [PATCH 1/8] feat(bigquery): support IAM conditions in datasets in Java client. --- .../java/com/google/cloud/bigquery/Acl.java | 85 ++++++++++++++++++- .../com/google/cloud/bigquery/BigQuery.java | 4 + .../google/cloud/bigquery/BigQueryImpl.java | 7 +- .../cloud/bigquery/spi/v2/BigQueryRpc.java | 3 +- .../bigquery/spi/v2/HttpBigQueryRpc.java | 15 ++++ .../com/google/cloud/bigquery/AclTest.java | 10 +++ .../cloud/bigquery/BigQueryImplTest.java | 15 ++++ .../cloud/bigquery/it/ITBigQueryTest.java | 22 +++++ 8 files changed, 153 insertions(+), 8 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Acl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Acl.java index e4107cdfd..27e6624f6 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Acl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Acl.java @@ -23,6 +23,7 @@ import com.google.api.services.bigquery.model.DatasetAccessEntry; import com.google.cloud.StringEnumType; import com.google.cloud.StringEnumValue; +import com.google.api.services.bigquery.model.Expr; import java.io.Serializable; import java.util.List; import java.util.Objects; @@ -41,6 +42,7 @@ public final class Acl implements Serializable { private final Entity entity; private final Role role; + private final Expr condition; /** * Dataset roles supported by BigQuery. @@ -568,9 +570,77 @@ Access toPb() { } } + /** + * Expr represents the conditional information related to dataset access policies. + */ + public static final class Expr implements Serializable { + // Textual representation of an expression in Common Expression Language syntax. + private final String expression; + /** + * Optional. Title for the expression, i.e. a short string describing + * its purpose. This can be used e.g. in UIs which allow to enter the expression. + */ + private final String title; + /** + * Optional. Description of the expression. This is a longer text which + * describes the expression, e.g. when hovered over it in a UI. + */ + private final String description; + /** + * Optional. String indicating the location of the expression for error + * reporting, e.g. a file name and a position in the file. + */ + private final String location; + + public Expr(String expression, String title, String description, String location) { + this.expression = expression; + this.title = title; + this.description = description; + this.location = location; + } + + com.google.api.services.bigquery.model.Expr toPb() { + com.google.api.services.bigquery.model.Expr bqExpr = new com.google.api.services.bigquery.model.Expr(); + bqExpr.setExpression(this.expression); + bqExpr.setTitle(this.title); + bqExpr.setDescription(this.description); + bqExpr.setLocation(this.location); + return bqExpr; + } + + static Expr fromPb(com.google.api.services.bigquery.model.Expr bqExpr) { + return new Expr(bqExpr.getExpression(), bqExpr.getTitle(), bqExpr.getDescription(), bqExpr.getLocation()); + } + + static Expr defaultExpr() { + return new Expr("", "", "", ""); + } + + @Override + public int hashCode() { + return Objects.hash(expression, title, description, location); + } + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + final Expr other = (Expr)obj; + return Objects.equals(this.expression, other.expression) && Objects.equals(this.title, other.title) && Objects.equals(this.description, other.description) && Objects.equals(this.location, other.location); + } + } + private Acl(Entity entity, Role role) { + this(entity, role, Expr.defaultExpr()); + } + + private Acl(Entity entity, Role role, Expr condition) { this.entity = checkNotNull(entity); this.role = role; + this.condition = condition; } /** @return Returns the entity for this ACL. */ @@ -582,6 +652,10 @@ public Entity getEntity() { public Role getRole() { return role; } + /** @return Returns the condition specified by this ACL. */ + public Expr getCondition() { + return condition; + } /** * @return Returns an Acl object. @@ -592,6 +666,10 @@ public static Acl of(Entity entity, Role role) { return new Acl(entity, role); } + public static Acl of(Entity entity, Role role, Expr condition) { + return new Acl(entity, role, condition); + } + /** * @param datasetAclEntity * @return Returns an Acl object for a datasetAclEntity. @@ -618,7 +696,7 @@ public static Acl of(Routine routine) { @Override public int hashCode() { - return Objects.hash(entity, role); + return Objects.hash(entity, role, condition); } @Override @@ -635,7 +713,7 @@ public boolean equals(Object obj) { return false; } final Acl other = (Acl) obj; - return Objects.equals(this.entity, other.entity) && Objects.equals(this.role, other.role); + return Objects.equals(this.entity, other.entity) && Objects.equals(this.role, other.role) && Objects.equals(this.condition, other.condition); } Access toPb() { @@ -643,11 +721,12 @@ Access toPb() { if (role != null) { accessPb.setRole(role.name()); } + accessPb.setCondition(condition.toPb()); return accessPb; } static Acl fromPb(Access access) { return Acl.of( - Entity.fromPb(access), access.getRole() != null ? Role.valueOf(access.getRole()) : null); + Entity.fromPb(access), access.getRole() != null ? Role.valueOf(access.getRole()) : null, Expr.fromPb(access.getCondition())); } } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java index 613134fa0..30024a1d0 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java @@ -289,6 +289,10 @@ public static DatasetOption fields(DatasetField... fields) { return new DatasetOption( BigQueryRpc.Option.FIELDS, Helper.selector(DatasetField.REQUIRED_FIELDS, fields)); } + + public static DatasetOption accessPolicyVersion(Integer accessPolicyVersion) { + return new DatasetOption(BigQueryRpc.Option.ACCESS_POLICY_VERSION, accessPolicyVersion); + } } /** Class for specifying dataset delete options. */ diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java index 770345000..dc1cf9d0d 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java @@ -261,10 +261,9 @@ public Page getNextPage() { public Dataset create(DatasetInfo datasetInfo, DatasetOption... options) { final com.google.api.services.bigquery.model.Dataset datasetPb = datasetInfo - .setProjectId( - Strings.isNullOrEmpty(datasetInfo.getDatasetId().getProject()) - ? getOptions().getProjectId() - : datasetInfo.getDatasetId().getProject()) + .setProjectId(Strings.isNullOrEmpty(datasetInfo.getDatasetId().getProject()) + ? getOptions().getProjectId() + : datasetInfo.getDatasetId().getProject()) .toPb(); final Map optionsMap = optionMap(options); try { diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java index 57f1a05c0..8b0a83531 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java @@ -59,7 +59,8 @@ enum Option { REQUESTED_POLICY_VERSION("requestedPolicyVersion"), TABLE_METADATA_VIEW("view"), RETRY_OPTIONS("retryOptions"), - BIGQUERY_RETRY_CONFIG("bigQueryRetryConfig"); + BIGQUERY_RETRY_CONFIG("bigQueryRetryConfig"), + ACCESS_POLICY_VERSION("accessPolicyVersion"); private final String value; diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java index 93337d8ca..2448f9d20 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java @@ -173,7 +173,22 @@ public Tuple> listDatasets(String projectId, Map options) { try { + Integer accessPolicyVersion = null; + for (Map.Entry entry : options.entrySet()) { + if (entry.getKey() == Option.ACCESS_POLICY_VERSION && entry.getValue() != null) { + accessPolicyVersion = (Integer)entry.getValue(); + } + } validateRPC(); + if (accessPolicyVersion != null) { + return bigquery + .datasets() + .insert(dataset.getDatasetReference().getProjectId(), dataset) + .setPrettyPrint(false) + .setFields(Option.FIELDS.getString(options)) + .setAccessPolicyVersion(accessPolicyVersion) + .execute(); + } return bigquery .datasets() .insert(dataset.getDatasetReference().getProjectId(), dataset) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/AclTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/AclTest.java index 30866c2b6..0b53f32ff 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/AclTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/AclTest.java @@ -23,6 +23,7 @@ import com.google.cloud.bigquery.Acl.Domain; import com.google.cloud.bigquery.Acl.Entity; import com.google.cloud.bigquery.Acl.Entity.Type; +import com.google.cloud.bigquery.Acl.Expr; import com.google.cloud.bigquery.Acl.Group; import com.google.cloud.bigquery.Acl.IamMember; import com.google.cloud.bigquery.Acl.Role; @@ -136,4 +137,13 @@ public void testOf() { assertEquals(routine, acl.getEntity()); assertEquals(null, acl.getRole()); } + + @Test + public void testOfWithCondition() { + Expr expr = new Expr("expression", "title", "description", "location"); + Acl acl = Acl.of(Group.ofAllAuthenticatedUsers(), Role.READER, expr); + Dataset.Access pb = acl.toPb(); + assertEquals(acl, Acl.fromPb(pb)); + assertEquals(acl.getCondition(), expr); + } } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryImplTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryImplTest.java index 88b8f6dbf..c13d272d2 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryImplTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryImplTest.java @@ -30,6 +30,7 @@ import com.google.cloud.RetryOption; import com.google.cloud.ServiceOptions; import com.google.cloud.Tuple; +import com.google.cloud.bigquery.BigQuery.DatasetOption; import com.google.cloud.bigquery.BigQuery.JobOption; import com.google.cloud.bigquery.BigQuery.QueryResultsOption; import com.google.cloud.bigquery.InsertAllRequest.RowToInsert; @@ -572,6 +573,20 @@ public void testCreateDatasetWithSelectedFields() { verify(bigqueryRpcMock).create(eq(DATASET_INFO_WITH_PROJECT.toPb()), capturedOptions.capture()); } + @Test + public void testCreateDatasetWithAccessPolicy() { + DatasetInfo datasetInfo = DATASET_INFO.setProjectId(OTHER_PROJECT); + DatasetOption datasetOption = DatasetOption.accessPolicyVersion(3); + when(bigqueryRpcMock.create(datasetInfo.toPb(), optionMap(datasetOption))) + .thenReturn(datasetInfo.toPb()); + BigQueryOptions bigQueryOptions = + createBigQueryOptionsForProject(OTHER_PROJECT, rpcFactoryMock); + bigquery = bigQueryOptions.getService(); + Dataset dataset = bigquery.create(datasetInfo, datasetOption); + assertEquals(new Dataset(bigquery, new DatasetInfo.BuilderImpl(datasetInfo)), dataset); + verify(bigqueryRpcMock).create(datasetInfo.toPb(), optionMap(datasetOption)); + } + @Test public void testGetDataset() { when(bigqueryRpcMock.getDataset(PROJECT, DATASET, EMPTY_RPC_OPTIONS)) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 84944d49c..d013e1830 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -41,6 +41,8 @@ import com.google.cloud.ServiceOptions; import com.google.cloud.bigquery.Acl; import com.google.cloud.bigquery.Acl.DatasetAclEntity; +import com.google.cloud.bigquery.Acl.Expr; +import com.google.cloud.bigquery.Acl.User; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQuery.DatasetDeleteOption; import com.google.cloud.bigquery.BigQuery.DatasetField; @@ -1688,6 +1690,26 @@ public void testCreateDatasetWithDefaultCollation() { RemoteBigQueryHelper.forceDelete(bigquery, collationDataset); } + @Test + public void testCreateDatabaseWithAccessPolicyVersion() { + String accessPolicyDataset = RemoteBigQueryHelper.generateDatasetName(); + User user = new User("Joe@example.com"); + Acl.Role role = Acl.Role.OWNER; + Acl.Expr condition = new Expr("expression", "title", "description", "location"); + Acl acl = Acl.of(user, role, condition); + DatasetInfo info = + DatasetInfo.newBuilder(accessPolicyDataset) + .setDescription(DESCRIPTION) + .setLabels(LABELS) + .setAcl(ImmutableList.of(acl)) + .build(); + DatasetOption datasetOption = DatasetOption.accessPolicyVersion(3); + Dataset dataset = bigquery.create(info, datasetOption); + assertNotNull(dataset); + + RemoteBigQueryHelper.forceDelete(bigquery, accessPolicyDataset); + } + @Test public void testCreateTableWithDefaultCollation() { String tableName = "test_create_table_with_default_collation"; From a79baf3b750877fee3e3d4cc58f06e4dae99c9c6 Mon Sep 17 00:00:00 2001 From: Liam Huffman Date: Mon, 9 Dec 2024 14:52:30 -0800 Subject: [PATCH 2/8] Fix formatting --- .../java/com/google/cloud/bigquery/Acl.java | 40 +++++++++++-------- .../google/cloud/bigquery/BigQueryImpl.java | 7 ++-- .../bigquery/spi/v2/HttpBigQueryRpc.java | 2 +- 3 files changed, 29 insertions(+), 20 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Acl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Acl.java index 27e6624f6..1e5b29f0f 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Acl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Acl.java @@ -21,9 +21,9 @@ import com.google.api.core.ApiFunction; import com.google.api.services.bigquery.model.Dataset.Access; import com.google.api.services.bigquery.model.DatasetAccessEntry; +import com.google.api.services.bigquery.model.Expr; import com.google.cloud.StringEnumType; import com.google.cloud.StringEnumValue; -import com.google.api.services.bigquery.model.Expr; import java.io.Serializable; import java.util.List; import java.util.Objects; @@ -570,25 +570,23 @@ Access toPb() { } } - /** - * Expr represents the conditional information related to dataset access policies. - */ + /** Expr represents the conditional information related to dataset access policies. */ public static final class Expr implements Serializable { // Textual representation of an expression in Common Expression Language syntax. private final String expression; /** - * Optional. Title for the expression, i.e. a short string describing - * its purpose. This can be used e.g. in UIs which allow to enter the expression. + * Optional. Title for the expression, i.e. a short string describing its purpose. This can be + * used e.g. in UIs which allow to enter the expression. */ private final String title; /** - * Optional. Description of the expression. This is a longer text which - * describes the expression, e.g. when hovered over it in a UI. + * Optional. Description of the expression. This is a longer text which describes the + * expression, e.g. when hovered over it in a UI. */ private final String description; /** - * Optional. String indicating the location of the expression for error - * reporting, e.g. a file name and a position in the file. + * Optional. String indicating the location of the expression for error reporting, e.g. a file + * name and a position in the file. */ private final String location; @@ -600,7 +598,8 @@ public Expr(String expression, String title, String description, String location } com.google.api.services.bigquery.model.Expr toPb() { - com.google.api.services.bigquery.model.Expr bqExpr = new com.google.api.services.bigquery.model.Expr(); + com.google.api.services.bigquery.model.Expr bqExpr = + new com.google.api.services.bigquery.model.Expr(); bqExpr.setExpression(this.expression); bqExpr.setTitle(this.title); bqExpr.setDescription(this.description); @@ -609,7 +608,8 @@ com.google.api.services.bigquery.model.Expr toPb() { } static Expr fromPb(com.google.api.services.bigquery.model.Expr bqExpr) { - return new Expr(bqExpr.getExpression(), bqExpr.getTitle(), bqExpr.getDescription(), bqExpr.getLocation()); + return new Expr( + bqExpr.getExpression(), bqExpr.getTitle(), bqExpr.getDescription(), bqExpr.getLocation()); } static Expr defaultExpr() { @@ -620,6 +620,7 @@ static Expr defaultExpr() { public int hashCode() { return Objects.hash(expression, title, description, location); } + @Override public boolean equals(Object obj) { if (this == obj) { @@ -628,8 +629,11 @@ public boolean equals(Object obj) { if (obj == null || getClass() != obj.getClass()) { return false; } - final Expr other = (Expr)obj; - return Objects.equals(this.expression, other.expression) && Objects.equals(this.title, other.title) && Objects.equals(this.description, other.description) && Objects.equals(this.location, other.location); + final Expr other = (Expr) obj; + return Objects.equals(this.expression, other.expression) + && Objects.equals(this.title, other.title) + && Objects.equals(this.description, other.description) + && Objects.equals(this.location, other.location); } } @@ -713,7 +717,9 @@ public boolean equals(Object obj) { return false; } final Acl other = (Acl) obj; - return Objects.equals(this.entity, other.entity) && Objects.equals(this.role, other.role) && Objects.equals(this.condition, other.condition); + return Objects.equals(this.entity, other.entity) + && Objects.equals(this.role, other.role) + && Objects.equals(this.condition, other.condition); } Access toPb() { @@ -727,6 +733,8 @@ Access toPb() { static Acl fromPb(Access access) { return Acl.of( - Entity.fromPb(access), access.getRole() != null ? Role.valueOf(access.getRole()) : null, Expr.fromPb(access.getCondition())); + Entity.fromPb(access), + access.getRole() != null ? Role.valueOf(access.getRole()) : null, + Expr.fromPb(access.getCondition())); } } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java index dc1cf9d0d..770345000 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java @@ -261,9 +261,10 @@ public Page getNextPage() { public Dataset create(DatasetInfo datasetInfo, DatasetOption... options) { final com.google.api.services.bigquery.model.Dataset datasetPb = datasetInfo - .setProjectId(Strings.isNullOrEmpty(datasetInfo.getDatasetId().getProject()) - ? getOptions().getProjectId() - : datasetInfo.getDatasetId().getProject()) + .setProjectId( + Strings.isNullOrEmpty(datasetInfo.getDatasetId().getProject()) + ? getOptions().getProjectId() + : datasetInfo.getDatasetId().getProject()) .toPb(); final Map optionsMap = optionMap(options); try { diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java index 2448f9d20..6fd1177f3 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java @@ -176,7 +176,7 @@ public Dataset create(Dataset dataset, Map options) { Integer accessPolicyVersion = null; for (Map.Entry entry : options.entrySet()) { if (entry.getKey() == Option.ACCESS_POLICY_VERSION && entry.getValue() != null) { - accessPolicyVersion = (Integer)entry.getValue(); + accessPolicyVersion = (Integer) entry.getValue(); } } validateRPC(); From 7610a8f36ea3c5bfcb957b895dd418aded920b57 Mon Sep 17 00:00:00 2001 From: Liam Huffman Date: Mon, 9 Dec 2024 18:36:39 -0800 Subject: [PATCH 3/8] Account for possible null condition field in Acl. --- .../java/com/google/cloud/bigquery/Acl.java | 12 +- .../bigquery/spi/v2/HttpBigQueryRpc.java | 32 +++++- .../cloud/bigquery/it/ITBigQueryTest.java | 105 +++++++++++++++++- 3 files changed, 139 insertions(+), 10 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Acl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Acl.java index 1e5b29f0f..c559504f7 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Acl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Acl.java @@ -612,10 +612,6 @@ static Expr fromPb(com.google.api.services.bigquery.model.Expr bqExpr) { bqExpr.getExpression(), bqExpr.getTitle(), bqExpr.getDescription(), bqExpr.getLocation()); } - static Expr defaultExpr() { - return new Expr("", "", "", ""); - } - @Override public int hashCode() { return Objects.hash(expression, title, description, location); @@ -638,7 +634,7 @@ public boolean equals(Object obj) { } private Acl(Entity entity, Role role) { - this(entity, role, Expr.defaultExpr()); + this(entity, role, null); } private Acl(Entity entity, Role role, Expr condition) { @@ -727,7 +723,9 @@ Access toPb() { if (role != null) { accessPb.setRole(role.name()); } - accessPb.setCondition(condition.toPb()); + if (condition != null) { + accessPb.setCondition(condition.toPb()); + } return accessPb; } @@ -735,6 +733,6 @@ static Acl fromPb(Access access) { return Acl.of( Entity.fromPb(access), access.getRole() != null ? Role.valueOf(access.getRole()) : null, - Expr.fromPb(access.getCondition())); + access.getCondition() != null ? Expr.fromPb(access.getCondition()) : null); } } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java index 6fd1177f3..5fa7c9e1f 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java @@ -130,6 +130,21 @@ private void validateRPC() throws BigQueryException, IOException { public Dataset getDataset(String projectId, String datasetId, Map options) { try { validateRPC(); + Integer accessPolicyVersion = null; + for (Map.Entry entry : options.entrySet()) { + if (entry.getKey() == Option.ACCESS_POLICY_VERSION && entry.getValue() != null) { + accessPolicyVersion = (Integer) entry.getValue(); + } + } + if (accessPolicyVersion != null) { + return bigquery + .datasets() + .get(projectId, datasetId) + .setFields(Option.FIELDS.getString(options)) + .setPrettyPrint(false) + .setAccessPolicyVersion(accessPolicyVersion) + .execute(); + } return bigquery .datasets() .get(projectId, datasetId) @@ -173,13 +188,13 @@ public Tuple> listDatasets(String projectId, Map options) { try { + validateRPC(); Integer accessPolicyVersion = null; for (Map.Entry entry : options.entrySet()) { if (entry.getKey() == Option.ACCESS_POLICY_VERSION && entry.getValue() != null) { accessPolicyVersion = (Integer) entry.getValue(); } } - validateRPC(); if (accessPolicyVersion != null) { return bigquery .datasets() @@ -291,7 +306,22 @@ public boolean deleteDataset(String projectId, String datasetId, Map public Dataset patch(Dataset dataset, Map options) { try { validateRPC(); + Integer accessPolicyVersion = null; + for (Map.Entry entry : options.entrySet()) { + if (entry.getKey() == Option.ACCESS_POLICY_VERSION && entry.getValue() != null) { + accessPolicyVersion = (Integer) entry.getValue(); + } + } DatasetReference reference = dataset.getDatasetReference(); + if (accessPolicyVersion != null) { + return bigquery + .datasets() + .patch(reference.getProjectId(), reference.getDatasetId(), dataset) + .setPrettyPrint(false) + .setFields(Option.FIELDS.getString(options)) + .setAccessPolicyVersion(accessPolicyVersion) + .execute(); + } return bigquery .datasets() .patch(reference.getProjectId(), reference.getDatasetId(), dataset) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index d013e1830..31f8ab347 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -1221,6 +1221,35 @@ public void testGetDatasetWithSelectedFields() { assertNull(dataset.getMaxTimeTravelHours()); } + @Test + public void testGetDatasetWithAccessPolicyVersion() { + String accessPolicyDataset = RemoteBigQueryHelper.generateDatasetName(); + User user = new User("liamhuffman@google.com"); + Acl.Role role = Acl.Role.WRITER; + Acl.Expr condition = + new Expr( + "request.time < timestamp('2030-01-01T00:00:00Z')", + "test condition", + "requests before the year 2030", + "location"); + Acl acl = Acl.of(user, role, condition); + DatasetOption datasetOption = DatasetOption.accessPolicyVersion(3); + + Dataset dataset = + bigquery.create( + DatasetInfo.newBuilder(accessPolicyDataset) + .setDescription("Some Description") + .setAcl(ImmutableList.of(acl)) + .build(), + datasetOption); + assertThat(dataset).isNotNull(); + + Dataset datasetClone = bigquery.getDataset(accessPolicyDataset, datasetOption); + assertNotNull(datasetClone); + assertEquals(dataset.getDescription(), datasetClone.getDescription()); + assertNotNull(datasetClone.getCreationTime()); + } + @Test public void testUpdateDataset() { Dataset dataset = @@ -1287,6 +1316,47 @@ public void testUpdateDatasetWithSelectedFields() { assertTrue(dataset.delete()); } + @Test + public void testUpdateDatabaseWithAccessPolicy() { + String accessPolicyDataset = RemoteBigQueryHelper.generateDatasetName(); + Dataset dataset = + bigquery.create( + DatasetInfo.newBuilder(accessPolicyDataset) + .setDescription("Some Description") + .setLabels(Collections.singletonMap("a", "b")) + .build()); + assertThat(dataset).isNotNull(); + + User user = new User("liamhuffman@google.com"); + Acl.Role role = Acl.Role.WRITER; + Acl.Expr condition = + new Expr( + "request.time < timestamp('2030-01-01T00:00:00Z')", + "test condition", + "requests before the year 2030", + "location"); + Acl acl = Acl.of(user, role, condition); + List acls = new ArrayList<>(); + acls.addAll(dataset.getAcl()); + acls.add(acl); + + DatasetOption datasetOption = DatasetOption.accessPolicyVersion(3); + Dataset updatedDataset = + bigquery.update( + dataset + .toBuilder() + .setDescription("Updated Description") + .setLabels(null) + .setAcl(acls) + .build(), + datasetOption); + assertNotNull(updatedDataset); + assertEquals(updatedDataset.getDescription(), "Updated Description"); + assertThat(updatedDataset.getLabels().isEmpty()); + + RemoteBigQueryHelper.forceDelete(bigquery, accessPolicyDataset); + } + @Test public void testGetNonExistingTable() { assertNull(bigquery.getTable(DATASET, "test_get_non_existing_table")); @@ -1693,9 +1763,14 @@ public void testCreateDatasetWithDefaultCollation() { @Test public void testCreateDatabaseWithAccessPolicyVersion() { String accessPolicyDataset = RemoteBigQueryHelper.generateDatasetName(); - User user = new User("Joe@example.com"); + User user = new User("liamhuffman@google.com"); Acl.Role role = Acl.Role.OWNER; - Acl.Expr condition = new Expr("expression", "title", "description", "location"); + Acl.Expr condition = + new Expr( + "request.time < timestamp('2030-01-01T00:00:00Z')", + "test condition", + "requests before the year 2030", + "location"); Acl acl = Acl.of(user, role, condition); DatasetInfo info = DatasetInfo.newBuilder(accessPolicyDataset) @@ -1706,6 +1781,32 @@ public void testCreateDatabaseWithAccessPolicyVersion() { DatasetOption datasetOption = DatasetOption.accessPolicyVersion(3); Dataset dataset = bigquery.create(info, datasetOption); assertNotNull(dataset); + assertEquals(dataset.getDescription(), DESCRIPTION); + + RemoteBigQueryHelper.forceDelete(bigquery, accessPolicyDataset); + } + + @Test(expected = BigQueryException.class) + public void testCreateDatabaseWithInvalidAccessPolicyVersion() { + String accessPolicyDataset = RemoteBigQueryHelper.generateDatasetName(); + User user = new User("liamhuffman@google.com"); + Acl.Role role = Acl.Role.READER; + Acl.Expr condition = + new Expr( + "request.time < timestamp('2030-01-01T00:00:00Z')", + "test condition", + "requests before the year 2030", + "location"); + Acl acl = Acl.of(user, role, condition); + DatasetInfo info = + DatasetInfo.newBuilder(accessPolicyDataset) + .setDescription(DESCRIPTION) + .setLabels(LABELS) + .setAcl(ImmutableList.of(acl)) + .build(); + DatasetOption datasetOption = DatasetOption.accessPolicyVersion(4); + Dataset dataset = bigquery.create(info, datasetOption); + assertNotNull(dataset); RemoteBigQueryHelper.forceDelete(bigquery, accessPolicyDataset); } From 001ed1ca1f23c412dc064ba728cab9a064d57500 Mon Sep 17 00:00:00 2001 From: Liam Huffman Date: Thu, 12 Dec 2024 16:33:12 -0800 Subject: [PATCH 4/8] Add toString() method to Acl.Expr object. Use service account in integration test instead of hardcoded personal account. Change Database API calls to only have one branch, toggling only the access policy version in a conditional. --- .../java/com/google/cloud/bigquery/Acl.java | 5 ++ .../bigquery/spi/v2/HttpBigQueryRpc.java | 64 ++++++++----------- .../cloud/bigquery/it/ITBigQueryTest.java | 12 ++-- 3 files changed, 37 insertions(+), 44 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Acl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Acl.java index c559504f7..8726eb440 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Acl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Acl.java @@ -631,6 +631,11 @@ public boolean equals(Object obj) { && Objects.equals(this.description, other.description) && Objects.equals(this.location, other.location); } + + @Override + public String toString() { + return toPb().toString(); + } } private Acl(Entity entity, Role role) { diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java index 5fa7c9e1f..bd90c3edb 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java @@ -136,21 +136,17 @@ public Dataset getDataset(String projectId, String datasetId, Map opt accessPolicyVersion = (Integer) entry.getValue(); } } + + Bigquery.Datasets.Get bqGetRequest = + bigquery + .datasets() + .get(projectId, datasetId) + .setFields(Option.FIELDS.getString(options)) + .setPrettyPrint(false); if (accessPolicyVersion != null) { - return bigquery - .datasets() - .get(projectId, datasetId) - .setFields(Option.FIELDS.getString(options)) - .setPrettyPrint(false) - .setAccessPolicyVersion(accessPolicyVersion) - .execute(); + bqGetRequest.setAccessPolicyVersion(accessPolicyVersion); } - return bigquery - .datasets() - .get(projectId, datasetId) - .setFields(Option.FIELDS.getString(options)) - .setPrettyPrint(false) - .execute(); + return bqGetRequest.execute(); } catch (IOException ex) { BigQueryException serviceException = translate(ex); if (serviceException.getCode() == HTTP_NOT_FOUND) { @@ -195,21 +191,16 @@ public Dataset create(Dataset dataset, Map options) { accessPolicyVersion = (Integer) entry.getValue(); } } + Bigquery.Datasets.Insert bqCreateRequest = + bigquery + .datasets() + .insert(dataset.getDatasetReference().getProjectId(), dataset) + .setPrettyPrint(false) + .setFields(Option.FIELDS.getString(options)); if (accessPolicyVersion != null) { - return bigquery - .datasets() - .insert(dataset.getDatasetReference().getProjectId(), dataset) - .setPrettyPrint(false) - .setFields(Option.FIELDS.getString(options)) - .setAccessPolicyVersion(accessPolicyVersion) - .execute(); + bqCreateRequest.setAccessPolicyVersion(accessPolicyVersion); } - return bigquery - .datasets() - .insert(dataset.getDatasetReference().getProjectId(), dataset) - .setPrettyPrint(false) - .setFields(Option.FIELDS.getString(options)) - .execute(); + return bqCreateRequest.execute(); } catch (IOException ex) { throw translate(ex); } @@ -313,21 +304,16 @@ public Dataset patch(Dataset dataset, Map options) { } } DatasetReference reference = dataset.getDatasetReference(); + Bigquery.Datasets.Patch bqPatchRequest = + bigquery + .datasets() + .patch(reference.getProjectId(), reference.getDatasetId(), dataset) + .setPrettyPrint(false) + .setFields(Option.FIELDS.getString(options)); if (accessPolicyVersion != null) { - return bigquery - .datasets() - .patch(reference.getProjectId(), reference.getDatasetId(), dataset) - .setPrettyPrint(false) - .setFields(Option.FIELDS.getString(options)) - .setAccessPolicyVersion(accessPolicyVersion) - .execute(); + bqPatchRequest.setAccessPolicyVersion(accessPolicyVersion); } - return bigquery - .datasets() - .patch(reference.getProjectId(), reference.getDatasetId(), dataset) - .setPrettyPrint(false) - .setFields(Option.FIELDS.getString(options)) - .execute(); + return bqPatchRequest.execute(); } catch (IOException ex) { throw translate(ex); } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 31f8ab347..2093e12da 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -213,6 +213,8 @@ public class ITBigQueryTest { private static final String MODEL_DATASET = RemoteBigQueryHelper.generateDatasetName(); private static final String ROUTINE_DATASET = RemoteBigQueryHelper.generateDatasetName(); private static final String PROJECT_ID = ServiceOptions.getDefaultProjectId(); + private static final String DEFAULT_USER_SERVICE_ACCOUNT = + "test-svc-bq-user@" + PROJECT_ID + ".iam.gserviceaccount.com"; private static final String RANDOM_ID = UUID.randomUUID().toString().substring(0, 8); private static final String STORAGE_BILLING_MODEL = "LOGICAL"; private static final Long MAX_TIME_TRAVEL_HOURS = 120L; @@ -1224,7 +1226,7 @@ public void testGetDatasetWithSelectedFields() { @Test public void testGetDatasetWithAccessPolicyVersion() { String accessPolicyDataset = RemoteBigQueryHelper.generateDatasetName(); - User user = new User("liamhuffman@google.com"); + User user = new User(DEFAULT_USER_SERVICE_ACCOUNT); Acl.Role role = Acl.Role.WRITER; Acl.Expr condition = new Expr( @@ -1317,7 +1319,7 @@ public void testUpdateDatasetWithSelectedFields() { } @Test - public void testUpdateDatabaseWithAccessPolicy() { + public void testUpdateDatabaseWithAccessPolicyVersion() { String accessPolicyDataset = RemoteBigQueryHelper.generateDatasetName(); Dataset dataset = bigquery.create( @@ -1327,7 +1329,7 @@ public void testUpdateDatabaseWithAccessPolicy() { .build()); assertThat(dataset).isNotNull(); - User user = new User("liamhuffman@google.com"); + User user = new User(DEFAULT_USER_SERVICE_ACCOUNT); Acl.Role role = Acl.Role.WRITER; Acl.Expr condition = new Expr( @@ -1763,7 +1765,7 @@ public void testCreateDatasetWithDefaultCollation() { @Test public void testCreateDatabaseWithAccessPolicyVersion() { String accessPolicyDataset = RemoteBigQueryHelper.generateDatasetName(); - User user = new User("liamhuffman@google.com"); + User user = new User(DEFAULT_USER_SERVICE_ACCOUNT); Acl.Role role = Acl.Role.OWNER; Acl.Expr condition = new Expr( @@ -1789,7 +1791,7 @@ public void testCreateDatabaseWithAccessPolicyVersion() { @Test(expected = BigQueryException.class) public void testCreateDatabaseWithInvalidAccessPolicyVersion() { String accessPolicyDataset = RemoteBigQueryHelper.generateDatasetName(); - User user = new User("liamhuffman@google.com"); + User user = new User(DEFAULT_USER_SERVICE_ACCOUNT); Acl.Role role = Acl.Role.READER; Acl.Expr condition = new Expr( From 15fc548c4496303406d267b06126fa24586f47ee Mon Sep 17 00:00:00 2001 From: Liam Huffman Date: Fri, 13 Dec 2024 11:45:25 -0800 Subject: [PATCH 5/8] Change Acl.User to be default google credentials in IT test --- .../cloud/bigquery/it/ITBigQueryTest.java | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 2093e12da..2cd82a5dd 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -42,6 +42,7 @@ import com.google.cloud.bigquery.Acl; import com.google.cloud.bigquery.Acl.DatasetAclEntity; import com.google.cloud.bigquery.Acl.Expr; +import com.google.cloud.bigquery.Acl.IamMember; import com.google.cloud.bigquery.Acl.User; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQuery.DatasetDeleteOption; @@ -213,8 +214,6 @@ public class ITBigQueryTest { private static final String MODEL_DATASET = RemoteBigQueryHelper.generateDatasetName(); private static final String ROUTINE_DATASET = RemoteBigQueryHelper.generateDatasetName(); private static final String PROJECT_ID = ServiceOptions.getDefaultProjectId(); - private static final String DEFAULT_USER_SERVICE_ACCOUNT = - "test-svc-bq-user@" + PROJECT_ID + ".iam.gserviceaccount.com"; private static final String RANDOM_ID = UUID.randomUUID().toString().substring(0, 8); private static final String STORAGE_BILLING_MODEL = "LOGICAL"; private static final Long MAX_TIME_TRAVEL_HOURS = 120L; @@ -1224,9 +1223,11 @@ public void testGetDatasetWithSelectedFields() { } @Test - public void testGetDatasetWithAccessPolicyVersion() { + public void testGetDatasetWithAccessPolicyVersion() throws IOException { String accessPolicyDataset = RemoteBigQueryHelper.generateDatasetName(); - User user = new User(DEFAULT_USER_SERVICE_ACCOUNT); + ServiceAccountCredentials credentials = + (ServiceAccountCredentials) GoogleCredentials.getApplicationDefault(); + User user = new User(credentials.getClientEmail()); Acl.Role role = Acl.Role.WRITER; Acl.Expr condition = new Expr( @@ -1319,8 +1320,10 @@ public void testUpdateDatasetWithSelectedFields() { } @Test - public void testUpdateDatabaseWithAccessPolicyVersion() { + public void testUpdateDatabaseWithAccessPolicyVersion() throws IOException { String accessPolicyDataset = RemoteBigQueryHelper.generateDatasetName(); + ServiceAccountCredentials credentials = + (ServiceAccountCredentials) GoogleCredentials.getApplicationDefault(); Dataset dataset = bigquery.create( DatasetInfo.newBuilder(accessPolicyDataset) @@ -1329,7 +1332,7 @@ public void testUpdateDatabaseWithAccessPolicyVersion() { .build()); assertThat(dataset).isNotNull(); - User user = new User(DEFAULT_USER_SERVICE_ACCOUNT); + User user = new User(credentials.getClientEmail()); Acl.Role role = Acl.Role.WRITER; Acl.Expr condition = new Expr( @@ -1763,9 +1766,11 @@ public void testCreateDatasetWithDefaultCollation() { } @Test - public void testCreateDatabaseWithAccessPolicyVersion() { + public void testCreateDatasetWithAccessPolicyVersion() throws IOException { String accessPolicyDataset = RemoteBigQueryHelper.generateDatasetName(); - User user = new User(DEFAULT_USER_SERVICE_ACCOUNT); + ServiceAccountCredentials credentials = + (ServiceAccountCredentials) GoogleCredentials.getApplicationDefault(); + User user = new User(credentials.getClientEmail()); Acl.Role role = Acl.Role.OWNER; Acl.Expr condition = new Expr( @@ -1789,9 +1794,11 @@ public void testCreateDatabaseWithAccessPolicyVersion() { } @Test(expected = BigQueryException.class) - public void testCreateDatabaseWithInvalidAccessPolicyVersion() { + public void testCreateDatabaseWithInvalidAccessPolicyVersion() throws IOException { String accessPolicyDataset = RemoteBigQueryHelper.generateDatasetName(); - User user = new User(DEFAULT_USER_SERVICE_ACCOUNT); + ServiceAccountCredentials credentials = + (ServiceAccountCredentials) GoogleCredentials.getApplicationDefault(); + User user = new User(credentials.getClientEmail()); Acl.Role role = Acl.Role.READER; Acl.Expr condition = new Expr( From 7babc4b350f01ad366c450e3075ccd7b98069c9a Mon Sep 17 00:00:00 2001 From: Liam Huffman Date: Fri, 13 Dec 2024 11:48:29 -0800 Subject: [PATCH 6/8] fix formatting --- .../test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 2cd82a5dd..8a8505c6a 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -42,7 +42,6 @@ import com.google.cloud.bigquery.Acl; import com.google.cloud.bigquery.Acl.DatasetAclEntity; import com.google.cloud.bigquery.Acl.Expr; -import com.google.cloud.bigquery.Acl.IamMember; import com.google.cloud.bigquery.Acl.User; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQuery.DatasetDeleteOption; From a17b6d99c6e107d406347a41e8db3a06be046da9 Mon Sep 17 00:00:00 2001 From: Liam Huffman Date: Wed, 18 Dec 2024 12:31:20 -0800 Subject: [PATCH 7/8] Add Acl.Expr builder. Fix review nits. --- .../java/com/google/cloud/bigquery/Acl.java | 69 ++++++++++++++++++- .../com/google/cloud/bigquery/BigQuery.java | 14 ++++ .../bigquery/spi/v2/HttpBigQueryRpc.java | 36 ++++------ .../cloud/bigquery/it/ITBigQueryTest.java | 53 ++++++++++---- 4 files changed, 134 insertions(+), 38 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Acl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Acl.java index 8726eb440..e6a2a0b91 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Acl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Acl.java @@ -590,6 +590,68 @@ public static final class Expr implements Serializable { */ private final String location; + private static final long serialVersionUID = 7358264726377291156L; + + static final class Builder { + private String expression; + private String title; + private String description; + private String location; + + Builder() {} + + Builder(Expr expr) { + this.expression = expr.expression; + this.title = expr.title; + this.description = expr.description; + this.location = expr.location; + } + + Builder(com.google.api.services.bigquery.model.Expr bqExpr) { + this.expression = bqExpr.getExpression(); + if (bqExpr.getTitle() != null) { + this.title = bqExpr.getTitle(); + } + if (bqExpr.getDescription() != null) { + this.description = bqExpr.getDescription(); + } + if (bqExpr.getLocation() != null) { + this.location = bqExpr.getLocation(); + } + } + + public Builder setExpression(String expression) { + this.expression = expression; + return this; + } + + public Builder setTitle(String title) { + this.title = title; + return this; + } + + public Builder setDescription(String description) { + this.description = description; + return this; + } + + public Builder setLocation(String location) { + this.location = location; + return this; + } + + public Expr build() { + return new Expr(this); + } + } + + public Expr(Builder builder) { + this.expression = builder.expression; + this.title = builder.title; + this.description = builder.description; + this.location = builder.location; + } + public Expr(String expression, String title, String description, String location) { this.expression = expression; this.title = title; @@ -608,8 +670,11 @@ com.google.api.services.bigquery.model.Expr toPb() { } static Expr fromPb(com.google.api.services.bigquery.model.Expr bqExpr) { - return new Expr( - bqExpr.getExpression(), bqExpr.getTitle(), bqExpr.getDescription(), bqExpr.getLocation()); + return new Builder(bqExpr).build(); + } + + public Builder toBuilder() { + return new Builder(this); } @Override diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java index 30024a1d0..e801b182b 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java @@ -290,6 +290,20 @@ public static DatasetOption fields(DatasetField... fields) { BigQueryRpc.Option.FIELDS, Helper.selector(DatasetField.REQUIRED_FIELDS, fields)); } + /** + * Returns an option to specify the dataset's access policy version for conditional access. If + * this option is not provided the field remains unset and conditional access cannot be used. + * Valid values are 0, 1, and 3. Requests specifying an invalid value will be rejected. Requests + * for conditional access policy binding in datasets must specify version 3. Datasets with no + * conditional role bindings in access policy may specify any valid value or leave the field + * unset. This field will be mapped to + * IAM Policy version and will + * be used to fetch the policy from IAM. If unset or if 0 or 1 the value is used for a dataset + * with conditional bindings, access entry with condition will have role string appended by + * 'withcond' string followed by a hash value. Please refer to + * + * Troubleshooting withcond for more details. + */ public static DatasetOption accessPolicyVersion(Integer accessPolicyVersion) { return new DatasetOption(BigQueryRpc.Option.ACCESS_POLICY_VERSION, accessPolicyVersion); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java index bd90c3edb..3946f83f5 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java @@ -130,12 +130,6 @@ private void validateRPC() throws BigQueryException, IOException { public Dataset getDataset(String projectId, String datasetId, Map options) { try { validateRPC(); - Integer accessPolicyVersion = null; - for (Map.Entry entry : options.entrySet()) { - if (entry.getKey() == Option.ACCESS_POLICY_VERSION && entry.getValue() != null) { - accessPolicyVersion = (Integer) entry.getValue(); - } - } Bigquery.Datasets.Get bqGetRequest = bigquery @@ -143,8 +137,10 @@ public Dataset getDataset(String projectId, String datasetId, Map opt .get(projectId, datasetId) .setFields(Option.FIELDS.getString(options)) .setPrettyPrint(false); - if (accessPolicyVersion != null) { - bqGetRequest.setAccessPolicyVersion(accessPolicyVersion); + for (Map.Entry entry : options.entrySet()) { + if (entry.getKey() == Option.ACCESS_POLICY_VERSION && entry.getValue() != null) { + bqGetRequest.setAccessPolicyVersion((Integer) entry.getValue()); + } } return bqGetRequest.execute(); } catch (IOException ex) { @@ -185,20 +181,16 @@ public Tuple> listDatasets(String projectId, Map options) { try { validateRPC(); - Integer accessPolicyVersion = null; - for (Map.Entry entry : options.entrySet()) { - if (entry.getKey() == Option.ACCESS_POLICY_VERSION && entry.getValue() != null) { - accessPolicyVersion = (Integer) entry.getValue(); - } - } Bigquery.Datasets.Insert bqCreateRequest = bigquery .datasets() .insert(dataset.getDatasetReference().getProjectId(), dataset) .setPrettyPrint(false) .setFields(Option.FIELDS.getString(options)); - if (accessPolicyVersion != null) { - bqCreateRequest.setAccessPolicyVersion(accessPolicyVersion); + for (Map.Entry entry : options.entrySet()) { + if (entry.getKey() == Option.ACCESS_POLICY_VERSION && entry.getValue() != null) { + bqCreateRequest.setAccessPolicyVersion((Integer) entry.getValue()); + } } return bqCreateRequest.execute(); } catch (IOException ex) { @@ -297,12 +289,6 @@ public boolean deleteDataset(String projectId, String datasetId, Map public Dataset patch(Dataset dataset, Map options) { try { validateRPC(); - Integer accessPolicyVersion = null; - for (Map.Entry entry : options.entrySet()) { - if (entry.getKey() == Option.ACCESS_POLICY_VERSION && entry.getValue() != null) { - accessPolicyVersion = (Integer) entry.getValue(); - } - } DatasetReference reference = dataset.getDatasetReference(); Bigquery.Datasets.Patch bqPatchRequest = bigquery @@ -310,8 +296,10 @@ public Dataset patch(Dataset dataset, Map options) { .patch(reference.getProjectId(), reference.getDatasetId(), dataset) .setPrettyPrint(false) .setFields(Option.FIELDS.getString(options)); - if (accessPolicyVersion != null) { - bqPatchRequest.setAccessPolicyVersion(accessPolicyVersion); + for (Map.Entry entry : options.entrySet()) { + if (entry.getKey() == Option.ACCESS_POLICY_VERSION && entry.getValue() != null) { + bqPatchRequest.setAccessPolicyVersion((Integer) entry.getValue()); + } } return bqPatchRequest.execute(); } catch (IOException ex) { diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 8a8505c6a..1f824a7dc 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -1230,9 +1230,9 @@ public void testGetDatasetWithAccessPolicyVersion() throws IOException { Acl.Role role = Acl.Role.WRITER; Acl.Expr condition = new Expr( - "request.time < timestamp('2030-01-01T00:00:00Z')", + "request.time > timestamp('2024-01-01T00:00:00Z')", "test condition", - "requests before the year 2030", + "requests after the year 2024", "location"); Acl acl = Acl.of(user, role, condition); DatasetOption datasetOption = DatasetOption.accessPolicyVersion(3); @@ -1246,10 +1246,21 @@ public void testGetDatasetWithAccessPolicyVersion() throws IOException { datasetOption); assertThat(dataset).isNotNull(); - Dataset datasetClone = bigquery.getDataset(accessPolicyDataset, datasetOption); - assertNotNull(datasetClone); - assertEquals(dataset.getDescription(), datasetClone.getDescription()); - assertNotNull(datasetClone.getCreationTime()); + Dataset remoteDataset = bigquery.getDataset(accessPolicyDataset, datasetOption); + assertNotNull(remoteDataset); + assertEquals(dataset.getDescription(), remoteDataset.getDescription()); + assertNotNull(remoteDataset.getCreationTime()); + + Acl remoteAclWithCond = null; + for (Acl remoteAcl : remoteDataset.getAcl()) { + if (remoteAcl.getCondition() != null) { + remoteAclWithCond = remoteAcl; + } + } + assertNotNull(remoteAclWithCond); + assertEquals(remoteAclWithCond.getCondition(), condition); + + RemoteBigQueryHelper.forceDelete(bigquery, accessPolicyDataset); } @Test @@ -1335,9 +1346,9 @@ public void testUpdateDatabaseWithAccessPolicyVersion() throws IOException { Acl.Role role = Acl.Role.WRITER; Acl.Expr condition = new Expr( - "request.time < timestamp('2030-01-01T00:00:00Z')", + "request.time > timestamp('2024-01-01T00:00:00Z')", "test condition", - "requests before the year 2030", + "requests after the year 2024", "location"); Acl acl = Acl.of(user, role, condition); List acls = new ArrayList<>(); @@ -1358,6 +1369,15 @@ public void testUpdateDatabaseWithAccessPolicyVersion() throws IOException { assertEquals(updatedDataset.getDescription(), "Updated Description"); assertThat(updatedDataset.getLabels().isEmpty()); + Acl updatedAclWithCond = null; + for (Acl updatedAcl : updatedDataset.getAcl()) { + if (updatedAcl.getCondition() != null) { + updatedAclWithCond = updatedAcl; + } + } + assertNotNull(updatedAclWithCond); + assertEquals(updatedAclWithCond.getCondition(), condition); + RemoteBigQueryHelper.forceDelete(bigquery, accessPolicyDataset); } @@ -1773,9 +1793,9 @@ public void testCreateDatasetWithAccessPolicyVersion() throws IOException { Acl.Role role = Acl.Role.OWNER; Acl.Expr condition = new Expr( - "request.time < timestamp('2030-01-01T00:00:00Z')", + "request.time > timestamp('2024-01-01T00:00:00Z')", "test condition", - "requests before the year 2030", + "requests after the year 2024", "location"); Acl acl = Acl.of(user, role, condition); DatasetInfo info = @@ -1789,6 +1809,15 @@ public void testCreateDatasetWithAccessPolicyVersion() throws IOException { assertNotNull(dataset); assertEquals(dataset.getDescription(), DESCRIPTION); + Acl remoteAclWithCond = null; + for (Acl remoteAcl : dataset.getAcl()) { + if (remoteAcl.getCondition() != null) { + remoteAclWithCond = remoteAcl; + } + } + assertNotNull(remoteAclWithCond); + assertEquals(remoteAclWithCond.getCondition(), condition); + RemoteBigQueryHelper.forceDelete(bigquery, accessPolicyDataset); } @@ -1801,9 +1830,9 @@ public void testCreateDatabaseWithInvalidAccessPolicyVersion() throws IOExceptio Acl.Role role = Acl.Role.READER; Acl.Expr condition = new Expr( - "request.time < timestamp('2030-01-01T00:00:00Z')", + "request.time > timestamp('2024-01-01T00:00:00Z')", "test condition", - "requests before the year 2030", + "requests after the year 2024", "location"); Acl acl = Acl.of(user, role, condition); DatasetInfo info = From fc6f203a361888bec390f7eda67854afb45457a0 Mon Sep 17 00:00:00 2001 From: Liam Huffman Date: Thu, 19 Dec 2024 10:39:43 -0800 Subject: [PATCH 8/8] fix formatting --- .../java/com/google/cloud/bigquery/BigQuery.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java index e801b182b..2a7d498c0 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java @@ -296,13 +296,13 @@ public static DatasetOption fields(DatasetField... fields) { * Valid values are 0, 1, and 3. Requests specifying an invalid value will be rejected. Requests * for conditional access policy binding in datasets must specify version 3. Datasets with no * conditional role bindings in access policy may specify any valid value or leave the field - * unset. This field will be mapped to - * IAM Policy version and will - * be used to fetch the policy from IAM. If unset or if 0 or 1 the value is used for a dataset - * with conditional bindings, access entry with condition will have role string appended by - * 'withcond' string followed by a hash value. Please refer to - * - * Troubleshooting withcond for more details. + * unset. This field will be mapped to IAM Policy version and will be + * used to fetch the policy from IAM. If unset or if 0 or 1 the value is used for a dataset with + * conditional bindings, access entry with condition will have role string appended by + * 'withcond' string followed by a hash value. Please refer to Troubleshooting + * withcond for more details. */ public static DatasetOption accessPolicyVersion(Integer accessPolicyVersion) { return new DatasetOption(BigQueryRpc.Option.ACCESS_POLICY_VERSION, accessPolicyVersion);