From f7e1cc0fcb1f2a1ea15d1f706600389daa7685ad Mon Sep 17 00:00:00 2001 From: Henry Avetisyan Date: Tue, 18 Jul 2017 12:40:19 -0700 Subject: [PATCH] #179 Use slave mysql servers for some of the read operations --- servers/zms/conf/zms.properties | 44 +++++- .../java/com/yahoo/athenz/zms/DBService.java | 143 ++++++++++-------- .../java/com/yahoo/athenz/zms/ZMSConsts.java | 9 +- .../java/com/yahoo/athenz/zms/ZMSImpl.java | 40 ++++- .../yahoo/athenz/zms/store/ObjectStore.java | 25 ++- .../zms/store/file/FileObjectStore.java | 2 +- .../zms/store/jdbc/JDBCObjectStore.java | 39 ++++- .../com/yahoo/athenz/zms/DBServiceTest.java | 7 +- .../com/yahoo/athenz/zms/ZMSImplTest.java | 6 +- .../zms/store/jdbc/JDBCObjectStoreTest.java | 55 ++++++- 10 files changed, 267 insertions(+), 103 deletions(-) diff --git a/servers/zms/conf/zms.properties b/servers/zms/conf/zms.properties index 87618c0d562..a5cfa8e31fe 100644 --- a/servers/zms/conf/zms.properties +++ b/servers/zms/conf/zms.properties @@ -125,19 +125,49 @@ athenz.zms.solution_templates_fname=${ROOT}/conf/zms_server/solution_templates.j # by the athenz.zms.file_store_name property #athenz.zms.file_store_name=zms_root -# JDBC URL where the ZMS Server will store domain json documents. -# jdbc:mysql://localhost:3306/zms - specifies MySQL instance +# JDBC URL where the ZMS Server will store domain data. +# The database server must be initialized with the ZMS +# server schema. For example, jdbc:mysql://localhost:3306/zms +# specifies a database called zms configured within a +# MySQL instance #athenz.zms.jdbc_store= -# If the jdbcstore is pointing to a MySQL server then this specifies -# the name of the user that has full access to the zms database +# If the athenz.zms.jdbc_store is pointing to a MySQL server then this +# specifies the name of the user that has full access to the configured +# ZMS server database #athenz.zms.jdbc_user= -# If the jdbcstore is pointing to a MySQL server then this specifies -# the password for the jdbc user that has been granted full access -# to the configured zms database +# If the athenz.zms.jdbc_store is pointing to a MySQL server then this +# specifies the password key for the jdbc user that has been granted full +# access to the configured ZMS server database. The configured +# private key store will be called with the value of the key to +# retrieve the password to authenticate requests against the +# configured MySQL server. #athenz.zms.jdbc_password= +# JDBC URL for slave databases that replicate ZMS Server's +# domain data. If configured, ZMS Server will use this database +# instance for any read only operation. It has the same syntax +# as the athenz.zms.jdbc_store property. +#athenz.zms.jdbc_ro_store= + +# If the athenz.zms.jdbc_ro_store is configured then this property is +# the name of the user that has full access to the zms database +# if this property is not specified but athenz.zms.jdbc_ro_store +# is configured, the server will use the value of the +# athenz.zms.jdbc_user property. +#athenz.zms.jdbc_ro_user= + +# If the athenz.zms.jdbc_ro_store is configured then this property +# specifies the password key for the jdbc user that has been granted +# full access to the configured zms database. If this property is not +# specified but athenz.zms.jdbc_ro_store is configured, the server +# will use the value of the athenz.zms.jdbc_password property. +# The configured private key store will be called with the value of +# the key to retrieve the password to authenticate requests against +# the configured MySQL server. +#athenz.zms.jdbc_ro_password= + # The number of seconds ZMS issued User Tokens are valid for #athenz.zms.user_token_timeout=3600 diff --git a/servers/zms/src/main/java/com/yahoo/athenz/zms/DBService.java b/servers/zms/src/main/java/com/yahoo/athenz/zms/DBService.java index b2e822cc45e..3afa2773dfa 100644 --- a/servers/zms/src/main/java/com/yahoo/athenz/zms/DBService.java +++ b/servers/zms/src/main/java/com/yahoo/athenz/zms/DBService.java @@ -72,10 +72,10 @@ public DBService(ObjectStore store, AuditLogger auditLogger, String userDomain) if (defaultOpTimeout < 0) { defaultOpTimeout = 60; } - if (store != null) { - store.setOperationTimeout(defaultOpTimeout); + if (this.store != null) { + this.store.setOperationTimeout(defaultOpTimeout); } - + // retrieve the concurrent update retry count. If we're given an invalid negative // value for count, we'll default back to our default configured value of 120 retries // which would result up to 30 seconds sleeping 250ms each time @@ -113,7 +113,7 @@ long getModTime() { } } - AthenzDomain getAthenzDomainFromCache(String domainName) { + AthenzDomain getAthenzDomainFromCache(String domainName, boolean masterCopy) { // if we have a match for a given domain name then we're going // to check if the last modified domain timestamp matches to what's @@ -129,7 +129,7 @@ AthenzDomain getAthenzDomainFromCache(String domainName) { } long modTime = 0; - try (ObjectStoreConnection con = store.getConnection(true)) { + try (ObjectStoreConnection con = store.getConnection(true, masterCopy)) { // we expect this response to come back immediately from // object store so we're going to use a smaller timeout @@ -150,7 +150,12 @@ AthenzDomain getAthenzDomainFromCache(String domainName) { } } - if (modTime == data.getModTime()) { + // if our cache data is same or newer than db then return + // data from the cache (it could be newer if we just updated + // the cache based on write db but during read, the server + // hasn't replicated the data yet) + + if (data.getModTime() >= modTime) { return data.getAthenzDomain(); } @@ -213,7 +218,7 @@ Domain makeDomain(ResourceContext ctx, String domainName, String description, St int retryCount = defaultRetryCount; do { - try (ObjectStoreConnection con = store.getConnection(false)) { + try (ObjectStoreConnection con = store.getConnection(false, true)) { // before adding this domain we need to verify our // quota check for sub-domains @@ -672,6 +677,8 @@ boolean shouldRetryOperation(ResourceException ex, int retryCount) { // this error indicates that the server is reporting is in // read-only mode which indicates a fail-over has taken place // and we need to clear all connections and start new ones + // this could only happen with write operations against the + // read-write object store store.clearConnections(); retry = true; @@ -704,7 +711,7 @@ void executePutPolicy(ResourceContext ctx, String domainName, String policyName, int retryCount = defaultRetryCount; do { - try (ObjectStoreConnection con = store.getConnection(false)) { + try (ObjectStoreConnection con = store.getConnection(false, true)) { // first verify that auditing requirements are met @@ -751,7 +758,7 @@ void executePutRole(ResourceContext ctx, String domainName, String roleName, Rol int retryCount = defaultRetryCount; do { - try (ObjectStoreConnection con = store.getConnection(false)) { + try (ObjectStoreConnection con = store.getConnection(false, true)) { // first verify that auditing requirements are met @@ -799,7 +806,7 @@ void executePutServiceIdentity(ResourceContext ctx, String domainName, String se int retryCount = defaultRetryCount; do { - try (ObjectStoreConnection con = store.getConnection(false)) { + try (ObjectStoreConnection con = store.getConnection(false, true)) { // first verify that auditing requirements are met @@ -846,7 +853,7 @@ void executePutPublicKeyEntry(ResourceContext ctx, String domainName, String ser int retryCount = defaultRetryCount; do { - try (ObjectStoreConnection con = store.getConnection(false)) { + try (ObjectStoreConnection con = store.getConnection(false, true)) { // first verify that auditing requirements are met @@ -913,7 +920,7 @@ void executeDeletePublicKeyEntry(ResourceContext ctx, String domainName, String int retryCount = defaultRetryCount; do { - try (ObjectStoreConnection con = store.getConnection(false)) { + try (ObjectStoreConnection con = store.getConnection(false, true)) { // first verify that auditing requirements are met @@ -976,7 +983,7 @@ void executePutMembership(ResourceContext ctx, String domainName, String roleNam int retryCount = defaultRetryCount; do { - try (ObjectStoreConnection con = store.getConnection(true)) { + try (ObjectStoreConnection con = store.getConnection(true, true)) { // first verify that auditing requirements are met @@ -1039,7 +1046,7 @@ void executePutEntity(ResourceContext ctx, String domainName, String entityName, int retryCount = defaultRetryCount; do { - try (ObjectStoreConnection con = store.getConnection(false)) { + try (ObjectStoreConnection con = store.getConnection(false, true)) { // first verify that auditing requirements are met @@ -1093,7 +1100,7 @@ void executeDeleteMembership(ResourceContext ctx, String domainName, String role int retryCount = defaultRetryCount; do { - try (ObjectStoreConnection con = store.getConnection(true)) { + try (ObjectStoreConnection con = store.getConnection(true, true)) { // first verify that auditing requirements are met @@ -1150,7 +1157,7 @@ void executeDeleteServiceIdentity(ResourceContext ctx, String domainName, String int retryCount = defaultRetryCount; do { - try (ObjectStoreConnection con = store.getConnection(false)) { + try (ObjectStoreConnection con = store.getConnection(false, true)) { // first verify that auditing requirements are met @@ -1188,7 +1195,7 @@ void executeDeleteEntity(ResourceContext ctx, String domainName, String entityNa int retryCount = defaultRetryCount; do { - try (ObjectStoreConnection con = store.getConnection(false)) { + try (ObjectStoreConnection con = store.getConnection(false, true)) { // first verify that auditing requirements are met @@ -1226,7 +1233,7 @@ void executeDeleteRole(ResourceContext ctx, String domainName, String roleName, int retryCount = defaultRetryCount; do { - try (ObjectStoreConnection con = store.getConnection(false)) { + try (ObjectStoreConnection con = store.getConnection(false, true)) { // first verify that auditing requirements are met @@ -1264,7 +1271,7 @@ void executeDeletePolicy(ResourceContext ctx, String domainName, String policyNa int retryCount = defaultRetryCount; do { - try (ObjectStoreConnection con = store.getConnection(false)) { + try (ObjectStoreConnection con = store.getConnection(false, true)) { // first verify that auditing requirements are met @@ -1322,7 +1329,7 @@ Domain executeDeleteDomain(ResourceContext ctx, String domainName, String auditR int retryCount = defaultRetryCount; do { - try (ObjectStoreConnection con = store.getConnection(false)) { + try (ObjectStoreConnection con = store.getConnection(false, true)) { // first verify that auditing requirements are met @@ -1354,7 +1361,7 @@ Domain executeDeleteDomain(ResourceContext ctx, String domainName, String auditR List listPrincipals(String domainName, boolean domainOnly) { - try (ObjectStoreConnection con = store.getConnection(true)) { + try (ObjectStoreConnection con = store.getConnection(true, false)) { List principals = con.listPrincipals(domainName); @@ -1484,7 +1491,7 @@ void executeDeleteUser(ResourceContext ctx, String userName, String auditRef, St int retryCount = defaultRetryCount; do { - try (ObjectStoreConnection con = store.getConnection(true)) { + try (ObjectStoreConnection con = store.getConnection(true, true)) { // remove all principal domains @@ -1518,6 +1525,8 @@ void executeDeleteUser(ResourceContext ctx, String userName, String auditRef, St auditLogRequest(ctx, userName, auditRef, caller, ZMSConsts.HTTP_DELETE, userName, null); + return; + } catch (ResourceException ex) { if (!shouldRetryOperation(ex, retryCount)) { throw ex; @@ -1529,14 +1538,14 @@ void executeDeleteUser(ResourceContext ctx, String userName, String auditRef, St ServiceIdentity getServiceIdentity(String domainName, String serviceName) { - try (ObjectStoreConnection con = store.getConnection(true)) { + try (ObjectStoreConnection con = store.getConnection(true, false)) { return getServiceIdentity(con, domainName, serviceName); } } DomainTemplateList listDomainTemplates(String domainName) { - try (ObjectStoreConnection con = store.getConnection(true)) { + try (ObjectStoreConnection con = store.getConnection(true, false)) { DomainTemplateList domainTemplateList = new DomainTemplateList(); domainTemplateList.setTemplateNames(con.listDomainTemplates(domainName)); return domainTemplateList; @@ -1594,7 +1603,7 @@ PublicKeyEntry getPublicKeyFromCache(String domainName, String serviceName, Stri PublicKeyEntry getServicePublicKeyEntry(String domainName, String serviceName, String keyId, boolean domainStateCheck) { - try (ObjectStoreConnection con = store.getConnection(true)) { + try (ObjectStoreConnection con = store.getConnection(true, false)) { return con.getPublicKeyEntry(domainName, serviceName, keyId, domainStateCheck); } catch (ResourceException ex) { if (ex.getCode() != ResourceException.SERVICE_UNAVAILABLE) { @@ -1621,7 +1630,7 @@ public ResourceAccessList getResourceAccessList(String principal, String action) // and needs to be optimized. For now we'll configure it with // default timeout of 30 minutes to avoid any issues - try (ObjectStoreConnection con = store.getConnection(true)) { + try (ObjectStoreConnection con = store.getConnection(true, false)) { con.setOperationTimeout(1800); return con.listResourceAccess(principal, action, userDomain); } @@ -1629,14 +1638,14 @@ public ResourceAccessList getResourceAccessList(String principal, String action) Domain getDomain(String domainName) { - try (ObjectStoreConnection con = store.getConnection(true)) { + try (ObjectStoreConnection con = store.getConnection(true, false)) { return con.getDomain(domainName); } } List listDomains(String prefix, long modifiedSince) { - try (ObjectStoreConnection con = store.getConnection(true)) { + try (ObjectStoreConnection con = store.getConnection(true, false)) { return con.listDomains(prefix, modifiedSince); } } @@ -1644,7 +1653,7 @@ List listDomains(String prefix, long modifiedSince) { DomainList lookupDomainById(String account, int productId) { DomainList domList = new DomainList(); - try (ObjectStoreConnection con = store.getConnection(true)) { + try (ObjectStoreConnection con = store.getConnection(true, false)) { String domain = con.lookupDomainById(account, productId); if (domain != null) { List list = Arrays.asList(domain); @@ -1665,7 +1674,7 @@ DomainList lookupDomainByProductId(Integer productId) { DomainList lookupDomainByRole(String roleMember, String roleName) { DomainList domList = new DomainList(); - try (ObjectStoreConnection con = store.getConnection(true)) { + try (ObjectStoreConnection con = store.getConnection(true, false)) { List domains = con.lookupDomainByRole(roleMember, roleName); if (domains != null) { domList.setNames(domains); @@ -1676,14 +1685,14 @@ DomainList lookupDomainByRole(String roleMember, String roleName) { List listRoles(String domainName) { - try (ObjectStoreConnection con = store.getConnection(true)) { + try (ObjectStoreConnection con = store.getConnection(true, false)) { return con.listRoles(domainName); } } Membership getMembership(String domainName, String roleName, String principal) { - try (ObjectStoreConnection con = store.getConnection(true)) { + try (ObjectStoreConnection con = store.getConnection(true, false)) { Membership membership = con.getRoleMember(domainName, roleName, principal); Timestamp expiration = membership.getExpiration(); @@ -1699,7 +1708,7 @@ Membership getMembership(String domainName, String roleName, String principal) { Role getRole(String domainName, String roleName, Boolean auditLog, Boolean expand) { - try (ObjectStoreConnection con = store.getConnection(true)) { + try (ObjectStoreConnection con = store.getConnection(true, false)) { return getRole(con, domainName, roleName, auditLog, expand); } } @@ -1753,7 +1762,7 @@ List getDelegatedRoleMembers(String domainName, String trustDomain, AthenzDomain domain = null; try { - domain = getAthenzDomain(trustDomain); + domain = getAthenzDomain(trustDomain, false); } catch (ResourceException ex) { } @@ -1816,24 +1825,24 @@ List getDelegatedRoleMembers(String domainName, String trustDomain, Policy getPolicy(String domainName, String policyName) { - try (ObjectStoreConnection con = store.getConnection(true)) { + try (ObjectStoreConnection con = store.getConnection(true, false)) { return getPolicy(con, domainName, policyName); } } Assertion getAssertion(String domainName, String policyName, Long assertionId) { - try (ObjectStoreConnection con = store.getConnection(true)) { + try (ObjectStoreConnection con = store.getConnection(true, false)) { return con.getAssertion(domainName, policyName, assertionId); } } - public void executePutAssertion(ResourceContext ctx, String domainName, String policyName, + void executePutAssertion(ResourceContext ctx, String domainName, String policyName, Assertion assertion, String auditRef, String caller) { int retryCount = defaultRetryCount; do { - try (ObjectStoreConnection con = store.getConnection(true)) { + try (ObjectStoreConnection con = store.getConnection(true, true)) { // first verify that auditing requirements are met @@ -1880,12 +1889,12 @@ public void executePutAssertion(ResourceContext ctx, String domainName, String p } while (retryCount > 0); } - public void executeDeleteAssertion(ResourceContext ctx, String domainName, String policyName, + void executeDeleteAssertion(ResourceContext ctx, String domainName, String policyName, Long assertionId, String auditRef, String caller) { int retryCount = defaultRetryCount; do { - try (ObjectStoreConnection con = store.getConnection(true)) { + try (ObjectStoreConnection con = store.getConnection(true, true)) { // first verify that auditing requirements are met @@ -1930,14 +1939,14 @@ public void executeDeleteAssertion(ResourceContext ctx, String domainName, Strin List listEntities(String domainName) { - try (ObjectStoreConnection con = store.getConnection(true)) { + try (ObjectStoreConnection con = store.getConnection(true, false)) { return con.listEntities(domainName); } } Entity getEntity(String domainName, String entityName) { - try (ObjectStoreConnection con = store.getConnection(true)) { + try (ObjectStoreConnection con = store.getConnection(true, false)) { return con.getEntity(domainName, entityName); } } @@ -1953,14 +1962,14 @@ Policy getPolicy(ObjectStoreConnection con, String domainName, String policyName List listPolicies(String domainName) { - try (ObjectStoreConnection con = store.getConnection(true)) { + try (ObjectStoreConnection con = store.getConnection(true, false)) { return con.listPolicies(domainName, null); } } List listServiceIdentities(String domainName) { - try (ObjectStoreConnection con = store.getConnection(true)) { + try (ObjectStoreConnection con = store.getConnection(true, false)) { return con.listServiceIdentities(domainName); } } @@ -1971,7 +1980,7 @@ void executePutDomainMeta(ResourceContext ctx, String domainName, DomainMeta met int retryCount = defaultRetryCount; Domain domain = null; do { - try (ObjectStoreConnection con = store.getConnection(false)) { + try (ObjectStoreConnection con = store.getConnection(false, true)) { // first verify that auditing requirements are met @@ -2040,7 +2049,7 @@ void executePutUserMeta(ResourceContext ctx, String userName, UserMeta meta, Str int retryCount = defaultRetryCount; do { - try (ObjectStoreConnection con = store.getConnection(false)) { + try (ObjectStoreConnection con = store.getConnection(false, true)) { // first we're going to retrieve the list domains for // the given principal @@ -2093,7 +2102,7 @@ void executePutDomainTemplate(ResourceContext ctx, String domainName, DomainTemp int retryCount = defaultRetryCount; do { - try (ObjectStoreConnection con = store.getConnection(false)) { + try (ObjectStoreConnection con = store.getConnection(false, true)) { // first verify that auditing requirements are met @@ -2141,7 +2150,7 @@ void executeDeleteDomainTemplate(ResourceContext ctx, String domainName, String int retryCount = defaultRetryCount; do { - try (ObjectStoreConnection con = store.getConnection(false)) { + try (ObjectStoreConnection con = store.getConnection(false, true)) { // first verify that auditing requirements are met @@ -2409,7 +2418,7 @@ void setupTenantAdminPolicy(ResourceContext ctx, String tenantDomain, String pro int retryCount = defaultRetryCount; do { - try (ObjectStoreConnection con = store.getConnection(false)) { + try (ObjectStoreConnection con = store.getConnection(false, true)) { // first verify that auditing requirements are met @@ -2487,7 +2496,7 @@ void executePutTenantRoles(ResourceContext ctx, String provSvcDomain, String pro int retryCount = defaultRetryCount; do { - try (ObjectStoreConnection con = store.getConnection(false)) { + try (ObjectStoreConnection con = store.getConnection(false, true)) { // first verify that auditing requirements are met @@ -2657,7 +2666,7 @@ void executePutProviderRoles(ResourceContext ctx, String tenantDomain, String pr int retryCount = defaultRetryCount; do { - try (ObjectStoreConnection con = store.getConnection(false)) { + try (ObjectStoreConnection con = store.getConnection(false, true)) { // first verify that auditing requirements are met @@ -2738,7 +2747,7 @@ void executeDeleteTenancy(ResourceContext ctx, String tenantDomain, String provS int retryCount = defaultRetryCount; do { - try (ObjectStoreConnection con = store.getConnection(false)) { + try (ObjectStoreConnection con = store.getConnection(false, true)) { // first verify that auditing requirements are met @@ -2819,7 +2828,7 @@ void executeDeleteTenantRoles(ResourceContext ctx, String provSvcDomain, String int retryCount = defaultRetryCount; do { - try (ObjectStoreConnection con = store.getConnection(false)) { + try (ObjectStoreConnection con = store.getConnection(false, true)) { // first verify that auditing requirements are met @@ -2890,14 +2899,14 @@ boolean isTrustRoleForTenant(ObjectStoreConnection con, String provSvcDomain, St boolean isTrustRoleForTenant(String provSvcDomain, String roleName, String rolePrefix, String tenantDomain) { - try (ObjectStoreConnection con = store.getConnection(true)) { + try (ObjectStoreConnection con = store.getConnection(true, false)) { return isTrustRoleForTenant(con, provSvcDomain, roleName, rolePrefix, tenantDomain); } } boolean isTenantRolePrefixMatch(String roleName, String rolePrefix, String tenantDomain) { - try (ObjectStoreConnection con = store.getConnection(true)) { + try (ObjectStoreConnection con = store.getConnection(true, false)) { return isTenantRolePrefixMatch(con, roleName, rolePrefix, tenantDomain); } } @@ -2962,16 +2971,16 @@ boolean isTenantRolePrefixMatch(ObjectStoreConnection con, String roleName, Stri return true; } - AthenzDomain getAthenzDomain(String domainName) { + AthenzDomain getAthenzDomain(String domainName, boolean masterCopy) { // first check to see if we our data is in the cache - AthenzDomain athenzDomain = getAthenzDomainFromCache(domainName); + AthenzDomain athenzDomain = getAthenzDomainFromCache(domainName, masterCopy); if (athenzDomain != null) { return athenzDomain; } - try (ObjectStoreConnection con = store.getConnection(true)) { + try (ObjectStoreConnection con = store.getConnection(true, masterCopy)) { athenzDomain = con.getAthenzDomain(domainName); setMembersInDomain(athenzDomain); } @@ -3006,7 +3015,12 @@ private void setMembersInDomain(AthenzDomain athenzDomain) { DomainModifiedList listModifiedDomains(long modifiedSince) { - try (ObjectStoreConnection con = store.getConnection(true)) { + // since this is the operation executed by ZTS servers to + // retrieve latest domain changes, we're going to use + // the read-write store as oppose to read-only store to + // get our up-to-date data + + try (ObjectStoreConnection con = store.getConnection(true, true)) { return con.listModifiedDomains(modifiedSince); } } @@ -3127,12 +3141,12 @@ void auditLogUserMeta(StringBuilder auditDetails, UserMeta meta) { .append("\"}"); } - public void executePutQuota(ResourceContext ctx, String domainName, Quota quota, + void executePutQuota(ResourceContext ctx, String domainName, Quota quota, String auditRef, String caller) { int retryCount = defaultRetryCount; do { - try (ObjectStoreConnection con = store.getConnection(true)) { + try (ObjectStoreConnection con = store.getConnection(true, true)) { // process our insert quota. since this is a "single" // operation, we are not using any transactions. @@ -3160,12 +3174,11 @@ public void executePutQuota(ResourceContext ctx, String domainName, Quota quota, } while (retryCount > 0); } - public void executeDeleteQuota(ResourceContext ctx, String domainName, - String auditRef, String caller) { + void executeDeleteQuota(ResourceContext ctx, String domainName, String auditRef, String caller) { int retryCount = defaultRetryCount; do { - try (ObjectStoreConnection con = store.getConnection(true)) { + try (ObjectStoreConnection con = store.getConnection(true, true)) { // process our delete quota request - it's a single // operation so no need to make it a transaction @@ -3191,7 +3204,7 @@ public void executeDeleteQuota(ResourceContext ctx, String domainName, } public Quota getQuota(String domainName) { - try (ObjectStoreConnection con = store.getConnection(true)) { + try (ObjectStoreConnection con = store.getConnection(true, false)) { return quotaCheck.getDomainQuota(con, domainName); } } diff --git a/servers/zms/src/main/java/com/yahoo/athenz/zms/ZMSConsts.java b/servers/zms/src/main/java/com/yahoo/athenz/zms/ZMSConsts.java index 4cb098665d1..96f63259fe9 100644 --- a/servers/zms/src/main/java/com/yahoo/athenz/zms/ZMSConsts.java +++ b/servers/zms/src/main/java/com/yahoo/athenz/zms/ZMSConsts.java @@ -36,9 +36,12 @@ public final class ZMSConsts { public static final String ZMS_PROP_CONFLICT_RETRY_COUNT = "athenz.zms.request_conflict_retry_count"; public static final String ZMS_PROP_CONFLICT_RETRY_SLEEP_TIME = "athenz.zms.request_conflict_retry_sleep_time"; - public static final String ZMS_PROP_JDBC_STORE = "athenz.zms.jdbc_store"; - public static final String ZMS_PROP_JDBC_USER = "athenz.zms.jdbc_user"; - public static final String ZMS_PROP_JDBC_PASSWORD = "athenz.zms.jdbc_password"; + public static final String ZMS_PROP_JDBC_RW_STORE = "athenz.zms.jdbc_store"; + public static final String ZMS_PROP_JDBC_RW_USER = "athenz.zms.jdbc_user"; + public static final String ZMS_PROP_JDBC_RW_PASSWORD = "athenz.zms.jdbc_password"; + public static final String ZMS_PROP_JDBC_RO_STORE = "athenz.zms.jdbc_ro_store"; + public static final String ZMS_PROP_JDBC_RO_USER = "athenz.zms.jdbc_ro_user"; + public static final String ZMS_PROP_JDBC_RO_PASSWORD = "athenz.zms.jdbc_ro_password"; public static final String ZMS_PROP_FILE_STORE_NAME = "athenz.zms.file_store_name"; public static final String ZMS_PROP_FILE_STORE_QUOTA = "athenz.zms.file_store_quota"; public static final String ZMS_PROP_FILE_STORE_PATH = "athenz.zms.file_store_path"; diff --git a/servers/zms/src/main/java/com/yahoo/athenz/zms/ZMSImpl.java b/servers/zms/src/main/java/com/yahoo/athenz/zms/ZMSImpl.java index 669e2958073..272b4305d6b 100644 --- a/servers/zms/src/main/java/com/yahoo/athenz/zms/ZMSImpl.java +++ b/servers/zms/src/main/java/com/yahoo/athenz/zms/ZMSImpl.java @@ -543,13 +543,35 @@ void loadConfigurationSettings() { void loadObjectStore() { ObjectStore store = null; - String jdbcStore = System.getProperty(ZMSConsts.ZMS_PROP_JDBC_STORE); + String jdbcStore = System.getProperty(ZMSConsts.ZMS_PROP_JDBC_RW_STORE); if (jdbcStore != null && jdbcStore.startsWith("jdbc:")) { - String jdbcUser = System.getProperty(ZMSConsts.ZMS_PROP_JDBC_USER); - String password = System.getProperty(ZMSConsts.ZMS_PROP_JDBC_PASSWORD, ""); + String jdbcUser = System.getProperty(ZMSConsts.ZMS_PROP_JDBC_RW_USER); + String password = System.getProperty(ZMSConsts.ZMS_PROP_JDBC_RW_PASSWORD, ""); String jdbcPassword = keyStore.getApplicationSecret(JDBC, password); - PoolableDataSource src = DataSourceFactory.create(jdbcStore, jdbcUser, jdbcPassword); - store = new JDBCObjectStore(src); + PoolableDataSource readWriteSrc = DataSourceFactory.create(jdbcStore, jdbcUser, jdbcPassword); + + // now check to see if we also have a read-only jdbc store configured + // if no username and password are specified then we'll use the + // read-write store credentials + + PoolableDataSource readOnlySrc = null; + String jdbcReadOnlyStore = System.getProperty(ZMSConsts.ZMS_PROP_JDBC_RO_STORE); + if (jdbcReadOnlyStore != null && jdbcReadOnlyStore.startsWith("jdbc:")) { + String jdbcReadOnlyUser = System.getProperty(ZMSConsts.ZMS_PROP_JDBC_RO_USER); + if (jdbcReadOnlyUser == null) { + jdbcReadOnlyUser = jdbcUser; + } + password = System.getProperty(ZMSConsts.ZMS_PROP_JDBC_RO_PASSWORD, ""); + String jdbcReadOnlyPassword = keyStore.getApplicationSecret(JDBC, password); + if (jdbcReadOnlyPassword == null) { + jdbcReadOnlyPassword = jdbcPassword; + } + readOnlySrc = DataSourceFactory.create(jdbcReadOnlyStore, + jdbcReadOnlyUser, jdbcReadOnlyPassword); + } + + store = new JDBCObjectStore(readWriteSrc, readOnlySrc); + } else { String homeDir = System.getProperty(ZMSConsts.ZMS_PROP_FILE_STORE_PATH, getRootDir() + "/var/zms_server"); @@ -1625,10 +1647,14 @@ boolean validRoleTokenAccess(String trustDomain, String domainName, String princ } AthenzDomain getAthenzDomain(String domainName, boolean ignoreExceptions) { + return getAthenzDomain(domainName, ignoreExceptions, false); + } + + AthenzDomain getAthenzDomain(String domainName, boolean ignoreExceptions, boolean masterCopy) { AthenzDomain domain = null; try { - domain = dbService.getAthenzDomain(domainName); + domain = dbService.getAthenzDomain(domainName, masterCopy); } catch (ResourceException ex) { if (LOG.isDebugEnabled()) { @@ -4029,7 +4055,7 @@ public void getSignedDomains(ResourceContext ctx, String domain, String metaOnly LOG.debug("getSignedDomains: retrieving domain " + dmod.getName()); } - AthenzDomain athenzDomain = getAthenzDomain(dmod.getName(), true); + AthenzDomain athenzDomain = getAthenzDomain(dmod.getName(), true, true); // it's possible that our domain was deleted by another // thread while we were processing this request so diff --git a/servers/zms/src/main/java/com/yahoo/athenz/zms/store/ObjectStore.java b/servers/zms/src/main/java/com/yahoo/athenz/zms/store/ObjectStore.java index 97d59a8f1f4..97980fa2643 100644 --- a/servers/zms/src/main/java/com/yahoo/athenz/zms/store/ObjectStore.java +++ b/servers/zms/src/main/java/com/yahoo/athenz/zms/store/ObjectStore.java @@ -17,12 +17,29 @@ public interface ObjectStore { - // get a new connection with the specified auto commit state - ObjectStoreConnection getConnection(boolean autoCommit); + /** + * Get a new connection from the object store with the specified + * auto commit state and read-only/write mode + * @param autoCommit connection will only used to make a single change + * so auto commit option should be set thus not requiring any explicit + * commit operations. + * @param readWrite the request is only for a read/write operation + * @return ObjectStoreConnection object + */ + ObjectStoreConnection getConnection(boolean autoCommit, boolean readWrite); - // operation timeout in seconds + /** + * Set the operation timeout for all requests + * @param opTimeout timeout in seconds + */ void setOperationTimeout(int opTimeout); - // clear all connections to the object store + /** + * Clear all connections to the object store. This is called when + * the server tries to write some object to the object store yet + * the store reports that it's not in write-only mode thus indicating + * it failed over to another master. So we need to clear all our + * connections and start new ones. + */ void clearConnections(); } diff --git a/servers/zms/src/main/java/com/yahoo/athenz/zms/store/file/FileObjectStore.java b/servers/zms/src/main/java/com/yahoo/athenz/zms/store/file/FileObjectStore.java index fed2a59251e..5b624223867 100644 --- a/servers/zms/src/main/java/com/yahoo/athenz/zms/store/file/FileObjectStore.java +++ b/servers/zms/src/main/java/com/yahoo/athenz/zms/store/file/FileObjectStore.java @@ -45,7 +45,7 @@ void verifyDirectory(File directory) { } @Override - public ObjectStoreConnection getConnection(boolean autoCommit) { + public ObjectStoreConnection getConnection(boolean autoCommit, boolean readWrite) { return new FileConnection(rootDir, quotaDir); } diff --git a/servers/zms/src/main/java/com/yahoo/athenz/zms/store/jdbc/JDBCObjectStore.java b/servers/zms/src/main/java/com/yahoo/athenz/zms/store/jdbc/JDBCObjectStore.java index 8155de50255..f977eb1a8ab 100644 --- a/servers/zms/src/main/java/com/yahoo/athenz/zms/store/jdbc/JDBCObjectStore.java +++ b/servers/zms/src/main/java/com/yahoo/athenz/zms/store/jdbc/JDBCObjectStore.java @@ -23,21 +23,43 @@ public class JDBCObjectStore implements ObjectStore { - PoolableDataSource src; + PoolableDataSource rwSrc; + PoolableDataSource roSrc; private int opTimeout = 60; //in seconds - public JDBCObjectStore(PoolableDataSource src) { - this.src = src; + public JDBCObjectStore(PoolableDataSource rwSrc, PoolableDataSource roSrc) { + this.rwSrc = rwSrc; + this.roSrc = roSrc; + + // if we're not given read-only source pool then we'll + // be using the read-write for all operations + + if (this.roSrc == null) { + this.roSrc = this.rwSrc; + } } @Override - public ObjectStoreConnection getConnection(boolean autoCommit) { + public ObjectStoreConnection getConnection(boolean autoCommit, boolean readWrite) { final String caller = "getConnection"; try { + PoolableDataSource src = readWrite ? rwSrc : roSrc; JDBCConnection jdbcConn = new JDBCConnection(src.getConnection(), autoCommit); jdbcConn.setOperationTimeout(opTimeout); return jdbcConn; } catch (Exception ex) { + + // if this was a read-only operation and we failed to get a connection + // then we're going to try to get a connection from our read-write + // pool first before throwing an exception + + if (!readWrite) { + return getConnection(autoCommit, true); + } + + // otherwise our service is not available and let the caller + // retry the request if necessary + throw ZMSUtils.error(ResourceException.SERVICE_UNAVAILABLE, ex.getMessage(), caller); } } @@ -47,8 +69,15 @@ public void setOperationTimeout(int opTimeout) { this.opTimeout = opTimeout; } + /** + * Clear all connections to the object store. This is called when + * the server tries to write some object to the database yet + * database reports that it's not in write-only mode thus indicating + * it failed over to another master. So we need to clear all our + * connections and start new ones. + */ @Override public void clearConnections() { - src.clearPoolConnections(); + rwSrc.clearPoolConnections(); } } diff --git a/servers/zms/src/test/java/com/yahoo/athenz/zms/DBServiceTest.java b/servers/zms/src/test/java/com/yahoo/athenz/zms/DBServiceTest.java index 0f1b0c2d311..05bdf37d6d9 100644 --- a/servers/zms/src/test/java/com/yahoo/athenz/zms/DBServiceTest.java +++ b/servers/zms/src/test/java/com/yahoo/athenz/zms/DBServiceTest.java @@ -169,7 +169,8 @@ private ZMSImpl zmsInit() { System.setProperty(ZMSConsts.ZMS_PROP_SOLUTION_TEMPLATE_FNAME, "src/test/resources/solution_templates.json"); System.setProperty(ZMSConsts.ZMS_PROP_FILE_STORE_PATH, "/tmp/zms_core_unit_tests/"); - System.clearProperty(ZMSConsts.ZMS_PROP_JDBC_STORE); + System.clearProperty(ZMSConsts.ZMS_PROP_JDBC_RW_STORE); + System.clearProperty(ZMSConsts.ZMS_PROP_JDBC_RO_STORE); ZMSImpl zmsObj = new ZMSImpl(); return zmsObj; @@ -1504,7 +1505,7 @@ public void testExecutePutPublicKeyEntryDisabledDomain() { UserMeta meta = new UserMeta().setEnabled(false); Domain domain = zms.dbService.getDomain(domainName); - ObjectStoreConnection con = zms.dbService.store.getConnection(false); + ObjectStoreConnection con = zms.dbService.store.getConnection(false, true); zms.dbService.updateUserDomainMeta(con, domain, meta); // this time the lookup should return nothing @@ -1602,7 +1603,7 @@ public void testExecutePutServiceIdentityRetryException() { "users", "host1"); Domain domain = new Domain().setAuditEnabled(false); - Mockito.when(mockObjStore.getConnection(false)).thenReturn(mockFileConn); + Mockito.when(mockObjStore.getConnection(false, true)).thenReturn(mockFileConn); Mockito.when(mockFileConn.getDomain(domainName)).thenReturn(domain); Mockito.when(mockFileConn.insertServiceIdentity(domainName, service)) .thenThrow(new ResourceException(ResourceException.CONFLICT, "conflict")); diff --git a/servers/zms/src/test/java/com/yahoo/athenz/zms/ZMSImplTest.java b/servers/zms/src/test/java/com/yahoo/athenz/zms/ZMSImplTest.java index 92a20168467..6a0b9dee629 100644 --- a/servers/zms/src/test/java/com/yahoo/athenz/zms/ZMSImplTest.java +++ b/servers/zms/src/test/java/com/yahoo/athenz/zms/ZMSImplTest.java @@ -231,7 +231,8 @@ private ZMSImpl zmsInit() { adminUser = System.getProperty(ZMSConsts.ZMS_PROP_DOMAIN_ADMIN); System.setProperty(ZMSConsts.ZMS_PROP_FILE_STORE_PATH, "/tmp/zms_core_unit_tests/"); - System.clearProperty(ZMSConsts.ZMS_PROP_JDBC_STORE); + System.clearProperty(ZMSConsts.ZMS_PROP_JDBC_RW_STORE); + System.clearProperty(ZMSConsts.ZMS_PROP_JDBC_RO_STORE); ZMSImpl zmsObj = new ZMSImpl(); zmsObj.serverPublicKeyMap.put("1", pubKeyK1); @@ -245,7 +246,8 @@ ZMSImpl getZmsImpl(String storeFile, AuditLogger alogger) { FileConnection.deleteDirectory(new File("/tmp/zms_core_unit_tests/" + storeFile)); - System.clearProperty(ZMSConsts.ZMS_PROP_JDBC_STORE); + System.clearProperty(ZMSConsts.ZMS_PROP_JDBC_RW_STORE); + System.clearProperty(ZMSConsts.ZMS_PROP_JDBC_RO_STORE); System.setProperty(ZMSConsts.ZMS_PROP_FILE_STORE_NAME, storeFile); System.setProperty(ZMSConsts.ZMS_PROP_FILE_STORE_PATH, "/tmp/zms_core_unit_tests/"); diff --git a/servers/zms/src/test/java/com/yahoo/athenz/zms/store/jdbc/JDBCObjectStoreTest.java b/servers/zms/src/test/java/com/yahoo/athenz/zms/store/jdbc/JDBCObjectStoreTest.java index 35f808a68d8..753196c2203 100644 --- a/servers/zms/src/test/java/com/yahoo/athenz/zms/store/jdbc/JDBCObjectStoreTest.java +++ b/servers/zms/src/test/java/com/yahoo/athenz/zms/store/jdbc/JDBCObjectStoreTest.java @@ -30,21 +30,64 @@ public class JDBCObjectStoreTest { @Test public void testGetConnection() throws SQLException { + + PoolableDataSource mockDataRwSrc = Mockito.mock(PoolableDataSource.class); + Connection rwMockConn = Mockito.mock(Connection.class); + Mockito.doReturn(rwMockConn).when(mockDataRwSrc).getConnection(); + + PoolableDataSource mockDataRoSrc = Mockito.mock(PoolableDataSource.class); + Connection roMockConn = Mockito.mock(Connection.class); + Mockito.doReturn(roMockConn).when(mockDataRoSrc).getConnection(); + + JDBCObjectStore store = new JDBCObjectStore(mockDataRwSrc, mockDataRoSrc); + + JDBCConnection jdbcConn = (JDBCConnection) store.getConnection(true, true); + assertEquals(jdbcConn.con, rwMockConn); + + jdbcConn = (JDBCConnection) store.getConnection(true, false); + assertEquals(jdbcConn.con, roMockConn); + } + + @Test + public void testGetConnectionReadWriteOnly() throws SQLException { PoolableDataSource mockDataSrc = Mockito.mock(PoolableDataSource.class); Connection mockConn = Mockito.mock(Connection.class); Mockito.doReturn(mockConn).when(mockDataSrc).getConnection(); - JDBCObjectStore store = new JDBCObjectStore(mockDataSrc); - assertNotNull(store.getConnection(true)); + JDBCObjectStore store = new JDBCObjectStore(mockDataSrc, null); + assertNotNull(store.getConnection(true, true)); + + // without read store we should also get a connection for a read + // only operation + assertNotNull(store.getConnection(true, true)); + store.clearConnections(); } @Test - public void testGetConnectionException() throws SQLException { + public void testGetConnectionReadOnly() throws SQLException { PoolableDataSource mockDataSrc = Mockito.mock(PoolableDataSource.class); - Mockito.doThrow(new SQLException()).when(mockDataSrc).getConnection(); + Connection mockConn = Mockito.mock(Connection.class); + Mockito.doReturn(mockConn).when(mockDataSrc).getConnection(); + JDBCObjectStore store = new JDBCObjectStore(null, mockDataSrc); + assertNotNull(store.getConnection(true, false)); + } + + @Test + public void testGetConnectionException() throws SQLException { + PoolableDataSource mockDataRwSrc = Mockito.mock(PoolableDataSource.class); + Mockito.doThrow(new SQLException()).when(mockDataRwSrc).getConnection(); + PoolableDataSource mockDataRoSrc = Mockito.mock(PoolableDataSource.class); + Mockito.doThrow(new SQLException()).when(mockDataRoSrc).getConnection(); + try { + JDBCObjectStore store = new JDBCObjectStore(mockDataRwSrc, mockDataRoSrc); + store.getConnection(true, true); + fail(); + } catch (RuntimeException ex) { + assertTrue(true); + } try { - JDBCObjectStore store = new JDBCObjectStore(mockDataSrc); - store.getConnection(true); + JDBCObjectStore store = new JDBCObjectStore(mockDataRwSrc, mockDataRoSrc); + store.getConnection(true, true); fail(); } catch (RuntimeException ex) { assertTrue(true);