From fa6ccbd6634ea8ea449824013b9a5fa32cb363d2 Mon Sep 17 00:00:00 2001 From: Che Molava Date: Thu, 22 Oct 2020 11:38:10 +0100 Subject: [PATCH 1/3] Explicitly set separator when building querystring rather than rely on the value of arg_separator.output. --- CRM/Mailchimp/Api3.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/CRM/Mailchimp/Api3.php b/CRM/Mailchimp/Api3.php index 6bb545f..2660389 100644 --- a/CRM/Mailchimp/Api3.php +++ b/CRM/Mailchimp/Api3.php @@ -56,7 +56,7 @@ class CRM_Mailchimp_Api3 { * Nb. a CiviCRM_Core_Error::debug_log_message facility is injected if you * enable debugging on the Mailchimp settings screen. But you can inject * something different, e.g. for testing. - */ + */ protected $log_facility; /** * @param array $settings contains key 'api_key', possibly other settings. @@ -69,10 +69,10 @@ public function __construct($settings) { } $this->api_key = $settings['api_key']; - // Set URL based on datacentre identifier at end of api key. + // Set URL based on datacentre identifier at end of api key. preg_match('/^.*-([^-]+)$/', $this->api_key, $matches); if (empty($matches[1])) { - throw new InvalidArgumentException("Invalid API key - could not extract datacentre from given API key."); + throw new InvalidArgumentException("Invalid API key - could not extract datacentre from given API key."); } if (!empty($settings['log_facility'])) { @@ -87,7 +87,7 @@ public function __construct($settings) { */ public function setLogFacility($callback) { if (!is_callable($callback)) { - throw new InvalidArgumentException("Log facility callback is not callable."); + throw new InvalidArgumentException("Log facility callback is not callable."); } $this->log_facility = $callback; } @@ -234,7 +234,7 @@ public function setNetworkEnabled($enable=TRUE) { /** * Provide mock for curl. * - * The callback will be called with the + * The callback will be called with the * request object in $this->request. It must return an array with optional * keys: * @@ -299,7 +299,7 @@ protected function makeRequest($method, $url, $data=null) { if ($this->request->method == 'GET') { // For GET requests, data must be added as query string. // Append if there's already a query string. - $query_string = http_build_query($data); + $query_string = http_build_query($data, '', '&'); if ($query_string) { $this->request->url .= ((strpos($this->request->url, '?')===false) ? '?' : '&') . $query_string; From e1bb69954fa61fd2693d3b57e08f87cecb3cf149 Mon Sep 17 00:00:00 2001 From: Che Molava Date: Thu, 22 Oct 2020 17:07:47 +0100 Subject: [PATCH 2/3] Add functionality to sync-push tags and profile fields. --- CRM/Mailchimp/Form/Setting.php | 132 ++++++++- CRM/Mailchimp/Form/Sync.php | 2 +- CRM/Mailchimp/Sync.php | 337 ++++++++++++++++++++--- mailchimp.php | 15 +- templates/CRM/Mailchimp/Form/Setting.tpl | 35 +++ 5 files changed, 465 insertions(+), 56 deletions(-) diff --git a/CRM/Mailchimp/Form/Setting.php b/CRM/Mailchimp/Form/Setting.php index ffd4fb2..aec35f9 100644 --- a/CRM/Mailchimp/Form/Setting.php +++ b/CRM/Mailchimp/Form/Setting.php @@ -42,9 +42,9 @@ public function buildQuickForm() { // Add the API Key Element $this->add('text', 'mailchimp_api_key', ts('API Key'), array( 'size' => 48, - ), TRUE); + ), TRUE); - // Add the User Security Key Element + // Add the User Security Key Element $this->add('text', 'mailchimp_security_key', ts('Security Key'), array( 'size' => 24, ), TRUE); @@ -53,6 +53,25 @@ public function buildQuickForm() { $enableOptions = array(1 => ts('Yes'), 0 => ts('No')); $this->addRadio('mailchimp_enable_debugging', ts('Enable Debugging'), $enableOptions, NULL); + + $result = civicrm_api3('UFGroup', 'get', array( + 'sequential' => 1, + 'group_type' => "Contact", + 'is_active' => 1, + )); + // Add profile selection for syncronization. + $profileOptions = array(0 => '-- None --'); + if (!empty($result['values'])) { + foreach ($result['values'] as $profile) { + $profileOptions[$profile['id']] = $profile['title']; + } + } + $yesNo = array(1 => ts('Yes'), 0 => ts('No')); + $this->addRadio('mailchimp_sync_checksum', ts('Sync Checksum and Contact ID'), $yesNo, NULL); + $this->add('select', 'mailchimp_sync_profile', ts('Sync fields from profile'), $profileOptions); + // Not a setting. + $this->addRadio('mailchimp_create_merge_fields', ts('Create missing fields on mailchimp lists'), $yesNo, NULL); + $this->addRadio('mailchimp_sync_tags', ts('Sync Tags?'), $yesNo, NULL); // Create the Submit Button. $buttons = array( array( @@ -62,7 +81,7 @@ public function buildQuickForm() { array( 'type' => 'cancel', 'name' => ts('Cancel'), - ), + ), ); // Add the Buttons. @@ -94,9 +113,16 @@ public function setDefaultValues() { } $enableDebugging = Civi::settings()->get('mailchimp_enable_debugging'); + $syncProfile = Civi::settings()->get('mailchimp_sync_profile'); + $syncChecksum = Civi::settings()->get('mailchimp_sync_checksum'); + $syncTags = Civi::settings()->get('mailchimp_sync_tags'); + $defaults['mailchimp_api_key'] = $apiKey; $defaults['mailchimp_security_key'] = $securityKey; $defaults['mailchimp_enable_debugging'] = $enableDebugging; + $defaults['mailchimp_sync_profile'] = $syncProfile; + $defaults['mailchimp_sync_checksum'] = $syncChecksum; + $defaults['mailchimp_sync_tags'] = $syncTags; return $defaults; } @@ -116,7 +142,14 @@ public function postProcess() { if (CRM_Utils_Array::value('mailchimp_api_key', $params) || CRM_Utils_Array::value('mailchimp_security_key', $params)) { - foreach (['mailchimp_api_key', 'mailchimp_enable_debugging', 'mailchimp_security_key'] as $_) { + foreach ([ + 'mailchimp_api_key', + 'mailchimp_enable_debugging', + 'mailchimp_security_key', + 'mailchimp_sync_checksum', + 'mailchimp_sync_tags', + 'mailchimp_sync_profile', + ] as $_) { Civi::settings()->set($_, $params[$_]); } @@ -143,7 +176,11 @@ public function postProcess() { // Check CMS's permission for (presumably) anonymous users. if (!self::checkMailchimpPermission($params['mailchimp_security_key'])) { CRM_Core_Session::setStatus(ts("Mailchimp WebHook URL requires 'allow webhook posts' permission to be set for any user roles.")); - } + } + // Create Merge Fields from for each list. + if (!empty($params['mailchimp_sync_profile']) && !empty($params['mailchimp_create_merge_fields'])) { + $this->createMailchimpMergeFields($params['mailchimp_sync_profile']); + } } } @@ -157,7 +194,7 @@ public static function checkMailchimpPermission($securityKey) { 'key' => $securityKey, ); $webhook_url = CRM_Utils_System::url('civicrm/mailchimp/webhook', $urlParams, TRUE, NULL, FALSE, TRUE); - + $curl = curl_init(); curl_setopt_array($curl, array( @@ -180,7 +217,88 @@ public static function checkMailchimpPermission($securityKey) { curl_close($curl); return ($info['http_code'] != 200) ? FALSE : TRUE; - } + } + + public function createMailchimpMergeFields($profileID, $includeChecksum=FALSE) { + // Get custom fields from profile and create data for creating on MC. + $ufFieldResult = civicrm_api3('UFField', 'get', ['uf_group_id' => $profileID]); + $mergeFields = $existingFields = array(); + if (!empty($ufFieldResult['values'])) { + foreach ($ufFieldResult['values'] as $field) { + if (0 === strpos($field['field_name'],'custom_') && $field['is_active']) { + $mergeFields[$field['field_name']] = [ + 'tag' => strtoupper($field['field_name']), + 'name' => $field['label'], + // By default make all fields type text and not public. + 'type' => 'text', + 'public' => FALSE, + ]; + } + } + } + // Checksum and contact id. + if (Civi::settings()->get('mailchimp_sync_checksum')) { + $default = ['type' => 'text', 'public' => FALSE]; + foreach ([ + 'contact_id' => 'Contact ID', + 'checksum' => 'Checksum' + + ] as $tag => $name) { + $mergeFields[$tag] = array_merge(['tag' => strtoupper($tag), 'name' => $name], $default); + } + } + + // Get existing fields for each list. + $listResult = civicrm_api3('Mailchimp', 'getlists'); + $msg = []; + if (!empty($listResult['values'])) { + foreach ($listResult['values'] as $listId => $listName) { + $listCreateFields = $mergeFields; + $existingFields = $this->getMergeFields($listId); + foreach($existingFields as $mergeField) { + $key = strtolower($mergeField->tag); + if (isset($listCreateFields[$key])) { + CRM_Core_Session::setStatus("Field $key exists on $listName, skipping"); + unset($listCreateFields[$key]); + } + } + // Create the merge field for the list. + foreach ($listCreateFields as $createField) { + $response = $this->createMergeField($listId, $createField); + if ($response && $response->http_code == 200) { + $name = $response->data->name; + $tag = $response->data->tag; + CRM_Core_Session::setStatus( "$name ($tag) created for $listName.", "Created field On Mailchimp"); + } + } + } + } + } + + /** + * Get Merge Field definitions for a Mailchimp Group. + * @param string $listId + * @return array + */ + protected function getMergeFields($listId) { + $mcClient = CRM_Mailchimp_Utils::getMailchimpApi(TRUE); + $path = '/lists/' . $listId . '/merge-fields'; + $response = $mcClient->get($path); + return $response->data->merge_fields; + } + + protected function createMergeField($listId, $data) { + $mcClient = CRM_Mailchimp_Utils::getMailchimpApi(TRUE); + $path = '/lists/' . $listId . '/merge-fields'; + try { + $response = $mcClient->post($path, $data); + } + catch (Exception $e) { + CRM_Core_Error::debug_log_message($e->getMessage()); + CRM_Core_Error::debug_var(__CLASS__ . __FUNCTION__, $response); + } + return $response; + } } diff --git a/CRM/Mailchimp/Form/Sync.php b/CRM/Mailchimp/Form/Sync.php index 89b616a..c77d1dc 100644 --- a/CRM/Mailchimp/Form/Sync.php +++ b/CRM/Mailchimp/Form/Sync.php @@ -313,7 +313,7 @@ public static function syncPushToMailchimp(CRM_Queue_TaskContext $ctx, $listID, // Finally, finish up by removing the two temporary tables //CRM_Mailchimp_Sync::dropTemporaryTables(); static::updatePushStats($stats); - + CRM_Mailchimp_Utils::checkDebug('End-CRM_Mailchimp_Form_Sync syncPushAdd $listID= ', $listID); return CRM_Queue_Task::TASK_SUCCESS; } diff --git a/CRM/Mailchimp/Sync.php b/CRM/Mailchimp/Sync.php index c2fb33c..ecd3b70 100644 --- a/CRM/Mailchimp/Sync.php +++ b/CRM/Mailchimp/Sync.php @@ -122,8 +122,8 @@ public function collectMailchimp($mode) { // Cheekily access the database directly to obtain a prepared statement. $db = $dao->getDatabaseConnection(); $insert = $db->prepare('INSERT INTO tmp_mailchimp_push_m - (email, first_name, last_name, hash, interests) - VALUES (?, ?, ?, ?, ?)'); + (email, first_name, last_name, hash, interests, checksum, addr1, addr2, city, state, zip, country, custom_fields, tags) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'); CRM_Mailchimp_Utils::checkDebug('CRM_Mailchimp_Form_Sync syncCollectMailchimp: ', $this->interest_group_details); @@ -140,12 +140,13 @@ public function collectMailchimp($mode) { $response = $api->get("/lists/$this->list_id/members", [ 'offset' => $offset, 'count' => $batch_size, 'status' => 'subscribed', - 'fields' => 'total_items,members.email_address,members.merge_fields,members.interests', + 'fields' => 'total_items,members.email_address,members.merge_fields,members.interests,members.tags' ]); $total = (int) $response->data->total_items; $offset += $batch_size; return $response->data->members; }; + $profileFields = self::getProfileFieldsToSync(); // // Main loop of all the records. @@ -166,6 +167,39 @@ public function collectMailchimp($mode) { $last_name = implode(' ', $names); } } + $checksum = $mode == 'push' && isset($member->merge_fields->CHECKSUM) ? $member->merge_fields->CHECKSUM : ''; + + // Address fields + $addr1 = $addr2 = $city = $state = $zip = $country = ''; + if (isset($member->merge_fields->ADDRESS)) { + $addr1 = $member->merge_fields->ADDRESS->addr1; + $addr2 = $member->merge_fields->ADDRESS->addr2; + $city = $member->merge_fields->ADDRESS->city; + $state = $member->merge_fields->ADDRESS->state; + $zip = $member->merge_fields->ADDRESS->zip; + $country = $member->merge_fields->ADDRESS->country; + } + // Custom merge fields. + $merge_fields = []; + foreach ($profileFields as $name => $field) { + $tag = strtoupper($name); + if (isset($member->merge_fields->{$tag})) { + $merge_fields[$tag] = $member->merge_fields->{$tag}; + } + } + $custom_fields = $merge_fields ? serialize($merge_fields) : ''; + // Member tags in Mailchimp + $mailchimpTags = []; + if (isset($member->tags)) { + foreach ($member->tags as $ignore => $tag) { + $mailchimpTags[] = $tag->name; + } + } + + $tags = ''; + if (!empty($mailchimpTags)) { + $tags = serialize(implode(',', $mailchimpTags)); + } // Find out which of our mapped groups apply to this subscriber. // Serialize the grouping array for SQL storage - this is the fastest way. @@ -193,6 +227,15 @@ public function collectMailchimp($mode) { $last_name, $hash, $interests, + $checksum, + $addr1, + $addr2, + $city, + $state, + $zip, + $country, + $custom_fields, + $tags ]); if ($result instanceof DB_Error) { throw new Exception ($result->message . "\n" . $result->userinfo); @@ -206,7 +249,7 @@ public function collectMailchimp($mode) { $db->freePrepared($insert); return $collected; } - + /** * Determine whether to push interest groups for any contacts sharing an email. * @return boolean @@ -214,8 +257,6 @@ public function collectMailchimp($mode) { public function isCollectGroupsByEmail() { foreach ($this->interest_group_details as $civi_group_id => $details) { if ($details['is_mc_update_grouping'] == 1) { - // If any interest group is configured to sallow updates from Mailchimp to - // CiviCRM, then aggregating will return FALSE; } } @@ -263,6 +304,22 @@ public function collectCiviCrm($mode) { // Use a nice API call to get the information for tmp_mailchimp_push_c. // The API will take care of smart groups. $start = microtime(TRUE); + $return_fields = [ + 'first_name', + 'last_name', + 'group', + 'street_address', + 'supplemental_address_1', + 'city', + 'state_province', + 'postal_code', + 'country', + 'tag' + ]; + $custom_fields = self::getProfileFieldsToSync(); + if ($custom_fields) { + $return_fields = array_merge($return_fields, array_keys($custom_fields)); + } $result = civicrm_api3('Contact', 'get', [ 'is_deleted' => 0, // The email filter in comment below does not work (CRM-18147) @@ -274,8 +331,11 @@ public function collectCiviCrm($mode) { 'do_not_email' => 0, 'on_hold' => 0, 'is_deceased' => 0, + // TODO: Unless the group is set as parent for the groups linked MC interest-groups, there is the possibility + // of contacts not being synched to the audience. + // Perhaps they should be included here. 'group' => $this->membership_group_id, - 'return' => ['first_name', 'last_name', 'group'], + 'return' => $return_fields, 'options' => ['limit' => 0], //'api.Email.get' => ['on_hold'=>0, 'return'=>'email,is_bulkmail'], ]); @@ -328,25 +388,25 @@ public function collectCiviCrm($mode) { } // Add email to contact. $result['values'][$id]['email'] = $email; - - + + if ($mode != 'push' || !$isGroupByEmail) { continue; } if (empty($groupsByEmail[$email])) { - // Groups are comma, separated string. + // Groups are comma-separated string. $groupsByEmail[$email] = $contact['groups']; } else { $groupsByEmail[$email] .= ',' . $contact['groups']; } } - + // Add data to temp table for comparison with mailchimp data. $start = microtime(TRUE); $collected = 0; - $insert = $db->prepare('INSERT IGNORE INTO tmp_mailchimp_push_c VALUES(?, ?, ?, ?, ?, ?)'); + $insert = $db->prepare('INSERT IGNORE INTO tmp_mailchimp_push_c VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'); // Note hashes being inserted to prevent db duplicate errors. $hashes = []; // Loop contacts: @@ -355,10 +415,22 @@ public function collectCiviCrm($mode) { continue; } $email = $contact['email']; - + $checksum = ''; + $contact_custom = []; + // Custom fields from profile. + foreach ($custom_fields as $name => $field) { + if (isset($contact[$name])) { + $tag = strtoupper($name); + $contact_custom[$tag] = $contact[$name]; + } + } + $custom_data = $contact_custom ? serialize($contact_custom) : ''; + + $tags = $contact['tags'] ? serialize($contact['tags']) : ''; + // Use the groups by email or use only the contact's groups. - $groups = !empty($groupsByEmail[$email]) - ? implode(',', array_unique(explode(',', $groupsByEmail[$email]))) + $groups = !empty($groupsByEmail[$email]) + ? implode(',', array_unique(explode(',', $groupsByEmail[$email]))) : $contact['groups']; $info = $this->getComparableInterestsFromCiviCrmGroups($groups, $mode); @@ -372,7 +444,7 @@ public function collectCiviCrm($mode) { // See note above about why we don't include email in the hash. // $hash = md5($email . $contact['first_name'] . $contact['last_name'] . $info); $hash = md5($contact['first_name'] . $contact['last_name'] . $info . $contact['id']); - + // Prevent duplicate rows db error. if (!empty($hashes[$hash . $email])) { continue; @@ -388,7 +460,16 @@ public function collectCiviCrm($mode) { $contact['first_name'], $contact['last_name'], $hash, - $info + $info, + $checksum, + $contact['street_address'], + $contact['supplemental_address_1'], + $contact['city'], + $contact['state_province_name'], + $contact['postal_code'], + $contact['country'], + $custom_data, + $tags )); } catch (PEAR_Exception $e) { @@ -540,7 +621,7 @@ public function matchMailchimpMembersToContacts() { public function mergeCiviInterestGroupsForDuplicateEmails() { // Retrieve duplicate emails with their interest groups. $sql = "SELECT c1.contact_id, c1.first_name, c1.last_name, c1.email, c1.interests, c1.contact_id, m.cid_guess - FROM tmp_mailchimp_push_c c1 + FROM tmp_mailchimp_push_c c1 LEFT JOIN mailchimp_push_m m ON c.contact_id = m.cid_guess WHERE c1.email IN ( SELECT c2.email FROM tmp_mailchimp_push_c c2 GROUP BY c2.email HAVING COUNT(distinct c2.interests) > 1) @@ -549,10 +630,10 @@ public function mergeCiviInterestGroupsForDuplicateEmails() { // Groups keyed by email. $groupsByEmail = []; // Contact representing the email (where a contact has already been mapped in mailchimp). - // We need these details to rebuild the hash with any group changes. + // We need these details to rebuild the hash with any group changes. $emailContacts = []; - - // Interest Group subscriptions for a contact is stored in the temp table as + + // Interest Group subscriptions for a contact is stored in the temp table as // an associative array of boolean values, keyed by the id of mc interest group. // To merge 2 sets of group, take the OR of each group. $mergeGroups = function ($g1, $g2) { @@ -562,8 +643,8 @@ public function mergeCiviInterestGroupsForDuplicateEmails() { } return $merged; }; - - // Collect the group data. + + // Collect the group data. while ($dao->fetch()) { if (!empty($dao->cid_guess)) { $emailContacts[$dao->email] = [ @@ -579,12 +660,12 @@ public function mergeCiviInterestGroupsForDuplicateEmails() { } } // Save to temp table. - foreach ($groupsByEmail as $email => $groups) { + foreach ($groupsByEmail as $email => $groups) { // Format the interests to store. $interests = serialize($groups); ksort($interests); $params = []; - // There is already a subscriber in mailchimp. + // There is already a subscriber in mailchimp. if (!empty($contactGroups[$email]['contact_id'])) { // Rebuild the hash. $contact = $contactGroups[$email]; @@ -598,13 +679,13 @@ public function mergeCiviInterestGroupsForDuplicateEmails() { ]; } else { - // Save interest groups + // Save interest groups $query = "UPDATE mailchimp_push_c SET interests = %1 WHERE email = %2"; $params = [ 1 => [$interests, 'String'], 2 => [$email, 'String'], ]; - } + } CRM_Core_DAO::executeQuery($query, $params); } } @@ -636,7 +717,7 @@ public function removeInSync($mode) { $doubles = 0; if ($mode == 'push') { $all_groups_by_email = []; - + $doubles = CRM_Mailchimp_Sync::runSqlReturnAffectedRows( 'DELETE c FROM tmp_mailchimp_push_c c @@ -681,12 +762,18 @@ public function updateMailchimpFromCivi() { $dao = CRM_Core_DAO::executeQuery( "SELECT c.interests c_interests, c.first_name c_first_name, c.last_name c_last_name, - c.email c_email, + c.email c_email, c.contact_id c_contact_id, c.checksum c_checksum, + c.addr1 c_addr1, c.addr2 c_addr2, c.city c_city, + c.state c_state, c.zip c_zip, c.country c_country, c.custom_fields c_custom_fields, c.tags c_tags, m.interests m_interests, m.first_name m_first_name, m.last_name m_last_name, - m.email m_email + m.email m_email, m.checksum m_checksum, m.addr1 m_addr1, m.addr2 m_addr2, m.city m_city, + m.state m_state, m.zip m_zip, m.country m_country, m.custom_fields m_custom_fields, m.tags m_tags FROM tmp_mailchimp_push_c c LEFT JOIN tmp_mailchimp_push_m m ON c.email = m.email;"); + // Check if syncTags is enabled in mailchmip setting. + $syncTag = Civi::settings()->get('mailchimp_sync_tags'); + $url_prefix = "/lists/$this->list_id/members/"; $changes = $additions = 0; // We need to know that the mailchimp list has certain merge fields. @@ -700,8 +787,8 @@ public function updateMailchimpFromCivi() { $params = static::updateMailchimpFromCiviLogic( $merge_fields, - ['email' => $dao->c_email, 'first_name' => $dao->c_first_name, 'last_name' => $dao->c_last_name, 'interests' => $dao->c_interests], - ['email' => $dao->m_email, 'first_name' => $dao->m_first_name, 'last_name' => $dao->m_last_name, 'interests' => $dao->m_interests]); + ['email' => $dao->c_email, 'first_name' => $dao->c_first_name, 'last_name' => $dao->c_last_name, 'interests' => $dao->c_interests, 'contact_id' => $dao->c_contact_id, 'checksum' => $dao->c_checksum, 'addr1' => $dao->c_addr1, 'addr2' => $dao->c_addr2, 'city' => $dao->c_city, 'state' => $dao->c_state, 'zip' => $dao->c_zip, 'country' => $dao->c_country, 'custom_fields' => $dao->c_custom_fields], + ['email' => $dao->m_email, 'first_name' => $dao->m_first_name, 'last_name' => $dao->m_last_name, 'interests' => $dao->m_interests, 'checksum' => $dao->m_checksum, 'addr1' => $dao->m_addr1, 'addr2' => $dao->m_addr2, 'city' => $dao->m_city, 'state' => $dao->m_state, 'zip' => $dao->m_zip, 'country' => $dao->m_country, 'custom_fields' => $dao->c_custom_fields]); if (!$params) { // This is the case if the changes could not be made due to policy @@ -711,6 +798,34 @@ public function updateMailchimpFromCivi() { continue; } + /* + Add or remove tags from a list member. + If a tag that does not exist is passed in and set as 'active', a new tag will be created in Mailchimp + */ + $tagsParams = array(); + + if ($syncTag == 1) { + $civiTags = $dao->c_tags ? explode(',', unserialize($dao->c_tags)) : array(); + $mailchimpTags = $dao->m_tags ? explode(',', unserialize($dao->m_tags)) : array(); + + // pushing all the tags from Civi to Mailchimp + foreach ($civiTags as $ignore => $tagName) { + $tagsParams[] = array( + 'name' => $tagName, + 'status' => 'active' + ); + } + + //Remove the tags in Mailchimp, if not in Civitags array (setting status inactive will remove the tag from the member) + foreach ($mailchimpTags as $ignore => $tagName) { + if (!in_array($tagName, $civiTags)) { + $tagsParams[] = array( + 'name' => $tagName, + 'status' => 'inactive' + ); + } + } + } if ($this->dry_run) { // Log the operation description. $_ = "Would " . ($dao->m_email ? 'update' : 'create') @@ -724,11 +839,20 @@ public function updateMailchimpFromCivi() { } } CRM_Mailchimp_Utils::checkDebug($_); + if (!empty($tagsParams)) { + $_tag = "Would update mailchimp member: $dao->m_email set tags = "; + CRM_Mailchimp_Utils::checkDebug($_tag, $tagsParams); + } } else { // Add the operation to the batch. $params['status'] = 'subscribed'; $operations[] = ['PUT', $url_prefix . md5(strtolower($dao->c_email)), $params]; + + if ($syncTag == 1 && !empty($tagsParams)) { + // Add POST tag operation to the batch. + $operations[] = ['POST', $url_prefix . md5(strtolower($dao->c_email)) . '/tags', array('tags' => $tagsParams)]; + } } if ($dao->m_email) { @@ -753,7 +877,7 @@ public function updateMailchimpFromCivi() { } else { // For real, not dry run. - // we are not unsubcribing Mailchimp members from CiviCRM now. + // We are not unsubcribing Mailchimp members from CiviCRM now since it is no longer reversible from the API. /* foreach ($removals as $email) { $operations[] = ['PATCH', $url_prefix . md5(strtolower($email)), ['status' => 'unsubscribed']]; @@ -1151,13 +1275,15 @@ public function countCiviCrmMembers() { * */ public function updateMailchimpFromCiviSingleContact($contact_id) { -return; + $customFields = self::getProfileFieldsToSync(); + $return = ['first_name', 'last_name', 'email_id', 'email', 'group', 'street_address', 'supplemental_address_1', 'city', 'state_province', 'postal_code', 'country', 'tag']; + $return = $customFields ? array_merge($return, array_keys($customFields)) : $return; // Get all the groups related to this list that the contact is currently in. // We have to use this dodgy API that concatenates the titles of the groups // with a comma (making it unsplittable if a group title has a comma in it). $contact = civicrm_api3('Contact', 'getsingle', [ 'contact_id' => $contact_id, - 'return' => ['first_name', 'last_name', 'email_id', 'email', 'group'], + 'return' => $return, 'sequential' => 1 ]); @@ -1214,6 +1340,34 @@ public function updateMailchimpFromCiviSingleContact($contact_id) { 'LNAME' => $contact['last_name'], ], ]; + // Add contact id and checksum merge fields. + $data['merge_fields']['CONTACT_ID'] = $contact_id; + $data['merge_fields']['CHECKSUM'] = CRM_Contact_BAO_Contact_Utils::generateChecksum($contact_id, NULL, 24 * 90); + + // Sync Address fields - addr1, city, state, zip, country are mandatory fields to update address in mailchimp. Passing 'n/a', if any of these fields are empty in CiviCRM + // To Do: Add a setting in CiviCRM to enable/disable sync of address fields and to set the default empty value + $addr1 = !empty($contact['street_address']) ? $contact['street_address'] : 'n/a'; + $addr2 = !empty($contact['supplemental_address_1']) ? $contact['supplemental_address_1'] : 'n/a'; + $city = !empty($contact['city']) ? $contact['city'] : 'n/a'; + $state = !empty($contact['state_province_name']) ? $contact['state_province_name'] : 'n/a'; + $zip = !empty($contact['postal_code']) ? $contact['postal_code'] : 'n/a'; + $country = !empty($contact['country']) ? $contact['country'] : 'n/a'; + $addressFields = array( + 'addr1' => $addr1, + 'addr2' => $addr2, + 'city' => $city, + 'state' => $state, + 'zip' => $zip, + 'country' => $country, + ); + $data['merge_fields']['ADDRESS'] = $addressFields; + foreach ($customFields as $name => $field) { + $tag = strtoupper($name); + $data['merge_fields'][$tag] = isset($contact[$name]) ? $contact[$name] : ''; + } + + // Do interest groups. + $data['interests'] = $this->getComparableInterestsFromCiviCrmGroups($contact['groups'], 'push'); // Do interest groups. $data['interests'] = $this->getComparableInterestsFromCiviCrmGroups($contact['groups'], 'push'); if (empty($data['interests'])) { @@ -1228,13 +1382,39 @@ public function updateMailchimpFromCiviSingleContact($contact_id) { catch (CRM_Mailchimp_NetworkErrorException $e) { CRM_Core_Session::setStatus(ts('There was a network problem trying to unsubscribe this contact at Mailchimp; any differences will remain until a CiviCRM to Mailchimp Sync is done.')); } + // check if syncTags is enabled in mailchimp setting + $syncTag = Civi::settings()->get('mailchimp_sync_tags'); + $tagsParams = array(); + if ($syncTag == 1) { + $civiTags = $contact['tags'] ? explode(',', $contact['tags']) : array(); + + // pushing all the tags from Civi to Mailchimp + // FIX ME : We are not checking and removing tags from Mailchimp here + foreach ($civiTags as $ignore => $tagName) { + $tagsParams[] = array( + 'name' => $tagName, + 'status' => 'active' + ); + } + + if (!empty($tagsParams)) { + try { + $result = $api->post("/lists/$this->list_id/members/$subscriber_hash/tags", array('tags' => $tagsParams)); + } catch (CRM_Mailchimp_RequestErrorException $e) { + CRM_Core_Session::setStatus(ts('There was a problem trying to sync tags of this contact at Mailchimp:') . $e->getMessage()); + } + catch (CRM_Mailchimp_NetworkErrorException $e) { + CRM_Core_Session::setStatus(ts('There was a network problem trying to add tags of this contact at Mailchimp; any differences will remain until a CiviCRM to Mailchimp Sync is done.')); + } + } + } } /** * Identify a contact who is expected to be subscribed to this list. * * This is used in a couple of cases, for finding a contact from incomming * data for: - * - a possibly new contact, + * - a possibly new contact, * - a contact that is expected to be in this membership group. * * Here's how we match a contact: @@ -1272,7 +1452,7 @@ public function updateMailchimpFromCiviSingleContact($contact_id) { * @param string|null $last_name * @param bool $must_be_on_list If TRUE, only return an ID if this contact * is known to be on the list. defaults to - * FALSE. + * FALSE. * @throw CRM_Mailchimp_DuplicateContactsException if the email is known bit * it fails to identify one contact. * @return int|null Contact Id if found. @@ -1414,13 +1594,13 @@ public static function guessContactIdsBySubscribers() { // First try matching on name and email. $r1 = static::runSqlReturnAffectedRows( "UPDATE tmp_mailchimp_push_m m - INNER JOIN tmp_mailchimp_push_c c - ON m.email = c.email + INNER JOIN tmp_mailchimp_push_c c + ON m.email = c.email AND m.first_name = c.first_name AND m.last_name = c.last_name SET m.cid_guess = c.contact_id WHERE m.cid_guess IS NULL"); - + // Now just email. return $r1 + static::runSqlReturnAffectedRows( "UPDATE tmp_mailchimp_push_m m @@ -1543,9 +1723,18 @@ public static function createTemporaryTableForMailchimp() { hash CHAR(32) NOT NULL DEFAULT '', interests VARCHAR(4096) NOT NULL DEFAULT '', cid_guess INT(10) DEFAULT NULL, + checksum VARCHAR(200) DEFAULT NULL, + addr1 varchar(96) DEFAULT NULL, + addr2 varchar(96) DEFAULT NULL, + city varchar(64) DEFAULT NULL, + state varchar(64) DEFAULT NULL, + zip varchar(64) DEFAULT NULL, + country varchar(64) DEFAULT NULL, + custom_fields VARCHAR(4096) NOT NULL DEFAULT '', + tags VARCHAR(4096) NOT NULL DEFAULT '', PRIMARY KEY (email, hash), KEY (cid_guess)) - ENGINE=InnoDB;"); + ENGINE=InnoDB ;"); // Convenience in collectMailchimp. return $dao; @@ -1565,6 +1754,15 @@ public static function createTemporaryTableForCiviCRM() { last_name VARCHAR(100) NOT NULL DEFAULT '', hash CHAR(32) NOT NULL DEFAULT '', interests VARCHAR(4096) NOT NULL DEFAULT '', + checksum VARCHAR(200) DEFAULT NULL, + addr1 varchar(96) DEFAULT NULL, + addr2 varchar(96) DEFAULT NULL, + city varchar(64) DEFAULT NULL, + state varchar(64) DEFAULT NULL, + zip varchar(64) DEFAULT NULL, + country varchar(64) DEFAULT NULL, + custom_fields VARCHAR(4096) NOT NULL DEFAULT '', + tags VARCHAR(4096) NOT NULL DEFAULT '', PRIMARY KEY (email, hash), KEY (contact_id) ) @@ -1630,7 +1828,40 @@ public static function updateMailchimpFromCiviLogic($merge_fields, $civi_details // names to this field. $params['merge_fields']['NAME'] = trim("$civi_details[first_name] $civi_details[last_name]"); } + // Does Mailchimp have a valid checksum for the contact? + if (!empty($merge_fields['CHECKSUM'])) { + $checksum = !empty($mailchimp_details['checksum']) ? $mailchimp_details['checksum'] : ''; + $checksum_is_valid = !empty($civi_details['contact_id']) && $checksum && CRM_Contact_BAO_Contact_Utils::validChecksum($civi_details['contact_id'], $checksum); + if (!$checksum_is_valid) { + // If the validation is too expensive, we may have to consider using an + // infinite time checksum. + $hours = 24 * 90; + $new_checksum = CRM_Contact_BAO_Contact_Utils::generateChecksum($civi_details['contact_id'], NULL, $hours); + $params['merge_fields']['CONTACT_ID'] = $civi_details['contact_id']; + $params['merge_fields']['CHECKSUM'] = $new_checksum; + } + } + // Sync Address fields - addr1, city, state, zip, country are mandatory fields to update address in mailchimp. Passing 'n/a', if any of these fields are empty in CiviCRM + // To Do: Add a setting in CiviCRM to enable/disable sync of address fields and to set the default empty value + if (isset($merge_fields['ADDRESS'])) { + $params['merge_fields']['ADDRESS']['addr1'] = !empty($civi_details['addr1']) ? $civi_details['addr1'] : 'n/a'; + $params['merge_fields']['ADDRESS']['addr2'] = !empty($civi_details['addr2']) ? $civi_details['addr2'] : 'n/a'; + $params['merge_fields']['ADDRESS']['city'] = !empty($civi_details['city']) ? $civi_details['city'] : 'n/a'; + $params['merge_fields']['ADDRESS']['state'] = !empty($civi_details['state']) ? $civi_details['state'] : 'n/a'; + $params['merge_fields']['ADDRESS']['zip'] = !empty($civi_details['zip']) ? $civi_details['zip'] : 'n/a'; + $params['merge_fields']['ADDRESS']['country'] = !empty($civi_details['country']) ? $civi_details['country'] : 'n/a'; + } + $profileFields = self::getProfileFieldsToSync(); + if ($profileFields && !empty($civi_details['custom_fields'])) { + $civiCustomData = unserialize($civi_details['custom_fields']); + foreach ($profileFields as $name => $ignore) { + $tag = strtoupper($name); + if (isset($merge_fields[$tag])) { + $params['merge_fields'][$tag] = $civiCustomData[$tag] ? $civiCustomData[$tag] : ''; + } + } + } return $params; } @@ -1672,5 +1903,29 @@ public static function runSqlReturnAffectedRows($sql, $params = array()) { $dao->free(); return $result; } + + /** + * Get the data for the fields to sync. + */ + public static function getProfileFieldsToSync() { + static $profileID; + if (!isset($profileID)) { + $profileID = Civi::settings()->get('mailchimp_sync_profile'); + } + static $fields = []; + $fields = []; + if (!$profileID) { + return $fields; + } + if (!$fields) { + $ufFieldResult = civicrm_api3('UFField', 'get', ['uf_group_id' => $profileID]); + if (!empty($ufFieldResult['values'])) { + foreach($ufFieldResult['values'] as $field) { + $fields[$field['field_name']] = $field; + } + } + } + return $fields; + } } diff --git a/mailchimp.php b/mailchimp.php index 3f87fc1..c81090a 100644 --- a/mailchimp.php +++ b/mailchimp.php @@ -46,7 +46,7 @@ function mailchimp_civicrm_install() { 'is_active' => 0, ); $result = civicrm_api3('job', 'create', $params); - + // Create Pull Sync job. $params = array( @@ -197,14 +197,14 @@ function mailchimp_civicrm_buildForm($formName, &$form) { $defaults['mc_integration_option'] = 0; } - $form->setDefaults($defaults); + $form->setDefaults($defaults); $form->assign('mailchimp_group_id' , $mcDetails[$groupId]['group_id']); $form->assign('mailchimp_list_id' , $mcDetails[$groupId]['list_id']); } else { // defaults for a new group $defaults['mc_integration_option'] = 0; $defaults['is_mc_update_grouping'] = 0; - $form->setDefaults($defaults); + $form->setDefaults($defaults); } } } @@ -358,7 +358,7 @@ function mailchimp_civicrm_pre( $op, $objectName, $id, &$params ) { ); if($objectName == 'Email') { - return; // @todo + return; // @todo // If about to delete an email in CiviCRM, we must delete it from Mailchimp // because we won't get chance to delete it once it's gone. // @@ -384,7 +384,7 @@ function mailchimp_civicrm_pre( $op, $objectName, $id, &$params ) { // If deleting an individual, delete their (bulk) email address from Mailchimp. if ($op == 'delete' && $objectName == 'Individual') { - return; // @todo + return; // @todo $result = civicrm_api('Contact', 'get', $params1); foreach ($result['values'] as $key => $value) { $emailId = $value['email_id']; @@ -487,7 +487,7 @@ function mailchimp_civicrm_post( $op, $objectName, $objectId, &$objectRef ) { // Post hook is disabled at this point in the running. return; } - + /***** NO BULK EMAILS (User Opt Out) *****/ if ($objectName == 'Individual' || $objectName == 'Organization' || $objectName == 'Household') { // Contact Edited @@ -530,7 +530,7 @@ function mailchimp_civicrm_post( $op, $objectName, $objectId, &$objectRef ) { /***** Contacts added/removed/deleted from CiviCRM group *****/ if ($objectName == 'GroupContact') { // Determine if the action being taken needs to affect Mailchimp at all. - + if ($op == 'view') { // Nothing changed; nothing to do. return; @@ -595,5 +595,6 @@ function mailchimp_civicrm_post( $op, $objectName, $objectId, &$objectRef ) { // Trigger mini sync for this person and this list. $sync = new CRM_Mailchimp_Sync($groups[$objectId]['list_id']); $sync->updateMailchimpFromCiviSingleContact($objectRef[0]); + CRM_Mailchimp_Utils::checkDebug('SyncSingleContact', $objectRef[0]); } } diff --git a/templates/CRM/Mailchimp/Form/Setting.tpl b/templates/CRM/Mailchimp/Form/Setting.tpl index ffeb1ad..006ea66 100644 --- a/templates/CRM/Mailchimp/Form/Setting.tpl +++ b/templates/CRM/Mailchimp/Form/Setting.tpl @@ -35,6 +35,41 @@ {$form.mailchimp_enable_debugging.html}
+ + {$form.mailchimp_sync_checksum.label} + {$form.mailchimp_sync_checksum.html}
+ + + {if $form.mailchimp_sync_profile} + + {$form.mailchimp_sync_profile.label} + {$form.mailchimp_sync_profile.html}
+ Optionally select a profile to include in contact syncronization. + These may be useful for segmentation, etc. on mailchimp.
Synchronizing custom fields will slow + down the process so only do so if necessary and use a profile with only the fields you need. + + + + + {$form.mailchimp_create_merge_fields.label} + {$form.mailchimp_create_merge_fields.html}
+ Select yes here to create merge fields on lists when this form is submitted. + By default the fields will be non-public text fields. + + + + {/if} + + {$form.mailchimp_sync_tags.label} + + {$form.mailchimp_sync_tags.html}
+ + {ts}Select 'Yes' here to sync CiviCRM contact tags with Mailchimp member tags during 'Sync Civi Contacts to Mailchimp' operation.
+ If a tag doesn't exist in mailchimp, new tag will be created in mailchimp and the member record will be updated with the new tag.
+ Existing tag will be removed from mailchimp member record, if the CiviCRM contact doesn't have it.{/ts} + + +
From d8329d03a889f875285991b82f2e87b50484095d Mon Sep 17 00:00:00 2001 From: Che Molava Date: Thu, 22 Oct 2020 17:44:42 +0100 Subject: [PATCH 3/3] Add descriptions for new settings. --- settings/mailchimp.setting.php | 35 ++++++++++++++++++++++++ templates/CRM/Mailchimp/Form/Setting.tpl | 3 +- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/settings/mailchimp.setting.php b/settings/mailchimp.setting.php index e357c4c..2cd001d 100644 --- a/settings/mailchimp.setting.php +++ b/settings/mailchimp.setting.php @@ -35,5 +35,40 @@ 'is_domain' => 1, 'is_contact' => 0, ], + 'mailchimp_sync_checksum' => [ + 'name' => 'mailchimp_sync_checksum', + 'title' => ts('Sync Checksum'), + 'description' => ts('If enabled will push CiviCRM Contact checksum, which then may be used in Mailchimp tokens.'), + 'group_name' => 'domain', + 'type' => 'Boolean', + 'default' => FALSE, + 'add' => '5.10', + 'is_domain' => 1, + 'is_contact' => 0, + ], + 'mailchimp_sync_tags' => [ + 'name' => 'mailchimp_sync_tags', + 'title' => ts('Sync Checksum'), + 'description' => ts('If enabled will push CiviCRM Contact tags, which then may be used for Mailchimp segmentation.'), + 'group_name' => 'domain', + 'type' => 'Boolean', + 'default' => FALSE, + 'add' => '5.10', + 'is_domain' => 1, + 'is_contact' => 0, + ], + 'mailchimp_sync_profile' => [ + 'name' => 'mailchimp_sync_profile', + 'title' => ts('Profile'), + 'description' => ts('Optionally select a profile to include in contact syncronization. + These may be useful for segmentation, etc. on mailchimp. Synchronizing custom fields will slow + down the process so only do so if necessary and use a profile with only the fields you need.'), + 'group_name' => 'domain', + 'type' => 'String', + 'default' => FALSE, + 'add' => '5.10', + 'is_domain' => 1, + 'is_contact' => 0, + ], ]; diff --git a/templates/CRM/Mailchimp/Form/Setting.tpl b/templates/CRM/Mailchimp/Form/Setting.tpl index 006ea66..9f0b8eb 100644 --- a/templates/CRM/Mailchimp/Form/Setting.tpl +++ b/templates/CRM/Mailchimp/Form/Setting.tpl @@ -44,9 +44,10 @@ {$form.mailchimp_sync_profile.label} {$form.mailchimp_sync_profile.html}
- Optionally select a profile to include in contact syncronization. + Optionally select a profile to include in contact push syncronization. These may be useful for segmentation, etc. on mailchimp.
Synchronizing custom fields will slow down the process so only do so if necessary and use a profile with only the fields you need. + Name, email and address fields will be synched by default so should not be included in the profile.