Merge pull request AthenZ#180 from yahoo/read

AthenZ#179 Use slave mysql servers for some of the read operations

servers/zms/conf/zms.properties

@@ -125,19 +125,49 @@ athenz.zms.solution_templates_fname=${ROOT}/conf/zms_server/solution_templates.j
 # by the athenz.zms.file_store_name property
 #athenz.zms.file_store_name=zms_root
 
-# JDBC URL where the ZMS Server will store domain json documents.
-# jdbc:mysql://localhost:3306/zms - specifies MySQL instance
+# JDBC URL where the ZMS Server will store domain data.
+# The database server must be initialized with the ZMS
+# server schema. For example, jdbc:mysql://localhost:3306/zms
+# specifies a database called zms configured within a
+# MySQL instance
 #athenz.zms.jdbc_store=
 
-# If the jdbcstore is pointing to a MySQL server then this specifies
-# the name of the user that has full access to the zms database
+# If the athenz.zms.jdbc_store is pointing to a MySQL server then this
+# specifies the name of the user that has full access to the configured
+# ZMS server database
 #athenz.zms.jdbc_user=
 
-# If the jdbcstore is pointing to a MySQL server then this specifies
-# the password for the jdbc user that has been granted full access
-# to the configured zms database
+# If the athenz.zms.jdbc_store is pointing to a MySQL server then this
+# specifies the password key for the jdbc user that has been granted full
+# access to the configured ZMS server database. The configured
+# private key store will be called with the value of the key to
+# retrieve the password to authenticate requests against the
+# configured MySQL server.
 #athenz.zms.jdbc_password=
 
+# JDBC URL for slave databases that replicate ZMS Server's
+# domain data. If configured, ZMS Server will use this database
+# instance for any read only operation. It has the same syntax
+# as the athenz.zms.jdbc_store property.
+#athenz.zms.jdbc_ro_store=
+
+# If the athenz.zms.jdbc_ro_store is configured then this property is
+# the name of the user that has full access to the zms database
+# if this property is not specified but athenz.zms.jdbc_ro_store
+# is configured, the server will use the value of the
+# athenz.zms.jdbc_user property.
+#athenz.zms.jdbc_ro_user=
+
+# If the athenz.zms.jdbc_ro_store is configured then this property
+# specifies the password key for the jdbc user that has been granted
+# full access to the configured zms database. If this property is not
+# specified but athenz.zms.jdbc_ro_store is configured, the server
+# will use the value of the athenz.zms.jdbc_password property.
+# The configured private key store will be called with the value of
+# the key to retrieve the password to authenticate requests against
+# the configured MySQL server.
+#athenz.zms.jdbc_ro_password=
+
 # The number of seconds ZMS issued User Tokens are valid for
 #athenz.zms.user_token_timeout=3600
 

servers/zms/src/main/java/com/yahoo/athenz/zms/ZMSConsts.java

@@ -36,9 +36,12 @@ public final class ZMSConsts {
     public static final String ZMS_PROP_CONFLICT_RETRY_COUNT      = "athenz.zms.request_conflict_retry_count";
     public static final String ZMS_PROP_CONFLICT_RETRY_SLEEP_TIME = "athenz.zms.request_conflict_retry_sleep_time";
 
-    public static final String ZMS_PROP_JDBC_STORE        = "athenz.zms.jdbc_store";
-    public static final String ZMS_PROP_JDBC_USER         = "athenz.zms.jdbc_user";
-    public static final String ZMS_PROP_JDBC_PASSWORD     = "athenz.zms.jdbc_password";
+    public static final String ZMS_PROP_JDBC_RW_STORE     = "athenz.zms.jdbc_store";
+    public static final String ZMS_PROP_JDBC_RW_USER      = "athenz.zms.jdbc_user";
+    public static final String ZMS_PROP_JDBC_RW_PASSWORD  = "athenz.zms.jdbc_password";
+    public static final String ZMS_PROP_JDBC_RO_STORE     = "athenz.zms.jdbc_ro_store";
+    public static final String ZMS_PROP_JDBC_RO_USER      = "athenz.zms.jdbc_ro_user";
+    public static final String ZMS_PROP_JDBC_RO_PASSWORD  = "athenz.zms.jdbc_ro_password";
     public static final String ZMS_PROP_FILE_STORE_NAME   = "athenz.zms.file_store_name";
     public static final String ZMS_PROP_FILE_STORE_QUOTA  = "athenz.zms.file_store_quota";
     public static final String ZMS_PROP_FILE_STORE_PATH   = "athenz.zms.file_store_path";

servers/zms/src/main/java/com/yahoo/athenz/zms/ZMSImpl.java

@@ -543,13 +543,35 @@ void loadConfigurationSettings() {
     void loadObjectStore() {
 
         ObjectStore store = null;
-        String jdbcStore = System.getProperty(ZMSConsts.ZMS_PROP_JDBC_STORE);
+        String jdbcStore = System.getProperty(ZMSConsts.ZMS_PROP_JDBC_RW_STORE);
         if (jdbcStore != null && jdbcStore.startsWith("jdbc:")) {
-            String jdbcUser = System.getProperty(ZMSConsts.ZMS_PROP_JDBC_USER);
-            String password = System.getProperty(ZMSConsts.ZMS_PROP_JDBC_PASSWORD, "");
+            String jdbcUser = System.getProperty(ZMSConsts.ZMS_PROP_JDBC_RW_USER);
+            String password = System.getProperty(ZMSConsts.ZMS_PROP_JDBC_RW_PASSWORD, "");
             String jdbcPassword = keyStore.getApplicationSecret(JDBC, password);
-            PoolableDataSource src = DataSourceFactory.create(jdbcStore, jdbcUser, jdbcPassword);
-            store = new JDBCObjectStore(src);
+            PoolableDataSource readWriteSrc = DataSourceFactory.create(jdbcStore, jdbcUser, jdbcPassword);
+
+            // now check to see if we also have a read-only jdbc store configured
+            // if no username and password are specified then we'll use the
+            // read-write store credentials
+
+            PoolableDataSource readOnlySrc = null;
+            String jdbcReadOnlyStore = System.getProperty(ZMSConsts.ZMS_PROP_JDBC_RO_STORE);
+            if (jdbcReadOnlyStore != null && jdbcReadOnlyStore.startsWith("jdbc:")) {
+                String jdbcReadOnlyUser = System.getProperty(ZMSConsts.ZMS_PROP_JDBC_RO_USER);
+                if (jdbcReadOnlyUser == null) {
+                    jdbcReadOnlyUser = jdbcUser;
+                }
+                password = System.getProperty(ZMSConsts.ZMS_PROP_JDBC_RO_PASSWORD, "");
+                String jdbcReadOnlyPassword = keyStore.getApplicationSecret(JDBC, password);
+                if (jdbcReadOnlyPassword == null) {
+                    jdbcReadOnlyPassword = jdbcPassword;
+                }
+                readOnlySrc = DataSourceFactory.create(jdbcReadOnlyStore,
+                        jdbcReadOnlyUser, jdbcReadOnlyPassword);
+            }
+
+            store = new JDBCObjectStore(readWriteSrc, readOnlySrc);
+
         } else {
             String homeDir = System.getProperty(ZMSConsts.ZMS_PROP_FILE_STORE_PATH,
                     getRootDir() + "/var/zms_server");
@@ -1625,10 +1647,14 @@ boolean validRoleTokenAccess(String trustDomain, String domainName, String princ
     }
 
     AthenzDomain getAthenzDomain(String domainName, boolean ignoreExceptions) {
+        return getAthenzDomain(domainName, ignoreExceptions, false);
+    }
+
+    AthenzDomain getAthenzDomain(String domainName, boolean ignoreExceptions, boolean masterCopy) {
 
         AthenzDomain domain = null;
         try {
-            domain = dbService.getAthenzDomain(domainName);
+            domain = dbService.getAthenzDomain(domainName, masterCopy);
         } catch (ResourceException ex) {
 
             if (LOG.isDebugEnabled()) {
@@ -4029,7 +4055,7 @@ public void getSignedDomains(ResourceContext ctx, String domain, String metaOnly
                 LOG.debug("getSignedDomains: retrieving domain " + dmod.getName());
             }
 
-            AthenzDomain athenzDomain = getAthenzDomain(dmod.getName(), true);
+            AthenzDomain athenzDomain = getAthenzDomain(dmod.getName(), true, true);
 
             // it's possible that our domain was deleted by another
             // thread while we were processing this request so

servers/zms/src/main/java/com/yahoo/athenz/zms/store/ObjectStore.java

@@ -17,12 +17,29 @@
 
 public interface ObjectStore {
 
-    // get a new connection with the specified auto commit state
-    ObjectStoreConnection getConnection(boolean autoCommit);
+    /**
+     * Get a new connection from the object store with the specified
+     * auto commit state and read-only/write mode
+     * @param autoCommit connection will only used to make a single change
+     * so auto commit option should be set thus not requiring any explicit
+     * commit operations.
+     * @param readWrite the request is only for a read/write operation
+     * @return ObjectStoreConnection object
+     */
+    ObjectStoreConnection getConnection(boolean autoCommit, boolean readWrite);
 
-    // operation timeout in seconds
+    /**
+     * Set the operation timeout for all requests
+     * @param opTimeout timeout in seconds
+     */
     void setOperationTimeout(int opTimeout);
 
-    // clear all connections to the object store
+    /**
+     * Clear all connections to the object store. This is called when
+     * the server tries to write some object to the object store yet
+     * the store reports that it's not in write-only mode thus indicating
+     * it failed over to another master. So we need to clear all our
+     * connections and start new ones.
+     */
     void clearConnections();
 }

servers/zms/src/main/java/com/yahoo/athenz/zms/store/file/FileObjectStore.java

@@ -45,7 +45,7 @@ void verifyDirectory(File directory) {
     }
 
     @Override
-    public ObjectStoreConnection getConnection(boolean autoCommit) {
+    public ObjectStoreConnection getConnection(boolean autoCommit, boolean readWrite) {
         return new FileConnection(rootDir, quotaDir);
     }
 

servers/zms/src/main/java/com/yahoo/athenz/zms/store/jdbc/JDBCObjectStore.java

@@ -23,21 +23,43 @@
 
 public class JDBCObjectStore implements ObjectStore {
 
-    PoolableDataSource src;
+    PoolableDataSource rwSrc;
+    PoolableDataSource roSrc;
     private int opTimeout = 60; //in seconds
 
-    public JDBCObjectStore(PoolableDataSource src) {
-        this.src = src;
+    public JDBCObjectStore(PoolableDataSource rwSrc, PoolableDataSource roSrc) {
+        this.rwSrc = rwSrc;
+        this.roSrc = roSrc;
+
+        // if we're not given read-only source pool then we'll
+        // be using the read-write for all operations
+
+        if (this.roSrc == null) {
+            this.roSrc = this.rwSrc;
+        }
     }
 
     @Override
-    public ObjectStoreConnection getConnection(boolean autoCommit) {
+    public ObjectStoreConnection getConnection(boolean autoCommit, boolean readWrite) {
         final String caller = "getConnection";
         try {
+            PoolableDataSource src = readWrite ? rwSrc : roSrc;
             JDBCConnection jdbcConn = new JDBCConnection(src.getConnection(), autoCommit);
             jdbcConn.setOperationTimeout(opTimeout);
             return jdbcConn;
         } catch (Exception ex) {
+
+            // if this was a read-only operation and we failed to get a connection
+            // then we're going to try to get a connection from our read-write
+            // pool first before throwing an exception
+
+            if (!readWrite) {
+                return getConnection(autoCommit, true);
+            }
+
+            // otherwise our service is not available and let the caller
+            // retry the request if necessary
+
             throw ZMSUtils.error(ResourceException.SERVICE_UNAVAILABLE, ex.getMessage(), caller);
         }
     }
@@ -47,8 +69,15 @@ public void setOperationTimeout(int opTimeout) {
         this.opTimeout = opTimeout;
     }
 
+    /**
+     * Clear all connections to the object store. This is called when
+     * the server tries to write some object to the database yet
+     * database reports that it's not in write-only mode thus indicating
+     * it failed over to another master. So we need to clear all our
+     * connections and start new ones.
+     */
     @Override
     public void clearConnections() {
-        src.clearPoolConnections();
+        rwSrc.clearPoolConnections();
     }
 }

servers/zms/src/test/java/com/yahoo/athenz/zms/DBServiceTest.java

@@ -169,7 +169,8 @@ private ZMSImpl zmsInit() {
         System.setProperty(ZMSConsts.ZMS_PROP_SOLUTION_TEMPLATE_FNAME, "src/test/resources/solution_templates.json");
 
         System.setProperty(ZMSConsts.ZMS_PROP_FILE_STORE_PATH, "/tmp/zms_core_unit_tests/");
-        System.clearProperty(ZMSConsts.ZMS_PROP_JDBC_STORE);
+        System.clearProperty(ZMSConsts.ZMS_PROP_JDBC_RW_STORE);
+        System.clearProperty(ZMSConsts.ZMS_PROP_JDBC_RO_STORE);
 
         ZMSImpl zmsObj = new ZMSImpl();
         return zmsObj;
@@ -1504,7 +1505,7 @@ public void testExecutePutPublicKeyEntryDisabledDomain() {
 
         UserMeta meta = new UserMeta().setEnabled(false);
         Domain domain = zms.dbService.getDomain(domainName);
-        ObjectStoreConnection con = zms.dbService.store.getConnection(false);
+        ObjectStoreConnection con = zms.dbService.store.getConnection(false, true);
         zms.dbService.updateUserDomainMeta(con, domain, meta);
 
         // this time the lookup should return nothing
@@ -1602,7 +1603,7 @@ public void testExecutePutServiceIdentityRetryException() {
                 "users", "host1");
 
         Domain domain = new Domain().setAuditEnabled(false);
-        Mockito.when(mockObjStore.getConnection(false)).thenReturn(mockFileConn);
+        Mockito.when(mockObjStore.getConnection(false, true)).thenReturn(mockFileConn);
         Mockito.when(mockFileConn.getDomain(domainName)).thenReturn(domain);
         Mockito.when(mockFileConn.insertServiceIdentity(domainName, service))
             .thenThrow(new ResourceException(ResourceException.CONFLICT, "conflict"));

servers/zms/src/test/java/com/yahoo/athenz/zms/ZMSImplTest.java

@@ -231,7 +231,8 @@ private ZMSImpl zmsInit() {
         adminUser = System.getProperty(ZMSConsts.ZMS_PROP_DOMAIN_ADMIN);
 
         System.setProperty(ZMSConsts.ZMS_PROP_FILE_STORE_PATH, "/tmp/zms_core_unit_tests/");
-        System.clearProperty(ZMSConsts.ZMS_PROP_JDBC_STORE);
+        System.clearProperty(ZMSConsts.ZMS_PROP_JDBC_RW_STORE);
+        System.clearProperty(ZMSConsts.ZMS_PROP_JDBC_RO_STORE);
 
         ZMSImpl zmsObj = new ZMSImpl();
         zmsObj.serverPublicKeyMap.put("1", pubKeyK1);
@@ -245,7 +246,8 @@ ZMSImpl getZmsImpl(String storeFile, AuditLogger alogger) {
 
         FileConnection.deleteDirectory(new File("/tmp/zms_core_unit_tests/" + storeFile));
 
-        System.clearProperty(ZMSConsts.ZMS_PROP_JDBC_STORE);
+        System.clearProperty(ZMSConsts.ZMS_PROP_JDBC_RW_STORE);
+        System.clearProperty(ZMSConsts.ZMS_PROP_JDBC_RO_STORE);
         System.setProperty(ZMSConsts.ZMS_PROP_FILE_STORE_NAME, storeFile);
         System.setProperty(ZMSConsts.ZMS_PROP_FILE_STORE_PATH, "/tmp/zms_core_unit_tests/");
 

servers/zms/src/test/java/com/yahoo/athenz/zms/store/jdbc/JDBCObjectStoreTest.java

@@ -30,21 +30,64 @@ public class JDBCObjectStoreTest {
 
     @Test
     public void testGetConnection() throws SQLException {
+
+        PoolableDataSource mockDataRwSrc = Mockito.mock(PoolableDataSource.class);
+        Connection rwMockConn = Mockito.mock(Connection.class);
+        Mockito.doReturn(rwMockConn).when(mockDataRwSrc).getConnection();
+
+        PoolableDataSource mockDataRoSrc = Mockito.mock(PoolableDataSource.class);
+        Connection roMockConn = Mockito.mock(Connection.class);
+        Mockito.doReturn(roMockConn).when(mockDataRoSrc).getConnection();
+
+        JDBCObjectStore store = new JDBCObjectStore(mockDataRwSrc, mockDataRoSrc);
+
+        JDBCConnection jdbcConn = (JDBCConnection) store.getConnection(true, true);
+        assertEquals(jdbcConn.con, rwMockConn);
+
+        jdbcConn = (JDBCConnection) store.getConnection(true, false);
+        assertEquals(jdbcConn.con, roMockConn);
+    }
+
+    @Test
+    public void testGetConnectionReadWriteOnly() throws SQLException {
         PoolableDataSource mockDataSrc = Mockito.mock(PoolableDataSource.class);
         Connection mockConn = Mockito.mock(Connection.class);
         Mockito.doReturn(mockConn).when(mockDataSrc).getConnection();
-        JDBCObjectStore store = new JDBCObjectStore(mockDataSrc);
-        assertNotNull(store.getConnection(true));
+        JDBCObjectStore store = new JDBCObjectStore(mockDataSrc, null);
+        assertNotNull(store.getConnection(true, true));
+
+        // without read store we should also get a connection for a read
+        // only operation
+        assertNotNull(store.getConnection(true, true));
+
         store.clearConnections();
     }
 
     @Test
-    public void testGetConnectionException() throws SQLException {
+    public void testGetConnectionReadOnly() throws SQLException {
         PoolableDataSource mockDataSrc = Mockito.mock(PoolableDataSource.class);
-        Mockito.doThrow(new SQLException()).when(mockDataSrc).getConnection();
+        Connection mockConn = Mockito.mock(Connection.class);
+        Mockito.doReturn(mockConn).when(mockDataSrc).getConnection();
+        JDBCObjectStore store = new JDBCObjectStore(null, mockDataSrc);
+        assertNotNull(store.getConnection(true, false));
+    }
+
+    @Test
+    public void testGetConnectionException() throws SQLException {
+        PoolableDataSource mockDataRwSrc = Mockito.mock(PoolableDataSource.class);
+        Mockito.doThrow(new SQLException()).when(mockDataRwSrc).getConnection();
+        PoolableDataSource mockDataRoSrc = Mockito.mock(PoolableDataSource.class);
+        Mockito.doThrow(new SQLException()).when(mockDataRoSrc).getConnection();
+        try {
+            JDBCObjectStore store = new JDBCObjectStore(mockDataRwSrc, mockDataRoSrc);
+            store.getConnection(true, true);
+            fail();
+        } catch (RuntimeException ex) {
+            assertTrue(true);
+        }
         try {
-            JDBCObjectStore store = new JDBCObjectStore(mockDataSrc);
-            store.getConnection(true);
+            JDBCObjectStore store = new JDBCObjectStore(mockDataRwSrc, mockDataRoSrc);
+            store.getConnection(true, true);
             fail();
         } catch (RuntimeException ex) {
             assertTrue(true);