diff --git a/lib/classes/class-ajax.php b/lib/classes/class-ajax.php index 8c3a3bf6d..3663c8007 100644 --- a/lib/classes/class-ajax.php +++ b/lib/classes/class-ajax.php @@ -118,37 +118,6 @@ public function action_stateless_wizard_update_settings($data) { wp_send_json(array('success' => true)); } - /** - * Fail over to image URL if not found on disk - * In case image not available on both local and bucket - * try to pull image from image URL in case it is accessible by some sort of proxy. - * - * @param: - * $url (int/string): URL of the image. - * $save_to (string): Path where to save the image. - * - * @return bool|int - * @throws \Exception - */ - public function get_attachment_if_exist($url, $save_to){ - if(is_int($url)) - $url = wp_get_attachment_url($url); - - $response = wp_remote_get( $url ); - if ( !is_wp_error($response) && is_array( $response ) ) { - if(!empty($response['response']['code']) && $response['response']['code'] == 200){ - try{ - if(wp_mkdir_p(dirname($save_to))){ - return file_put_contents($save_to, $response['body']); - } - } - catch(\Exception $e){ - throw $e; - } - } - } - return false; - } /** * Regenerate image sizes. @@ -180,8 +149,8 @@ public function action_stateless_process_image() { $result_code = ud_get_stateless_media()->get_client()->get_media( apply_filters( 'wp_stateless_file_name', str_replace( trailingslashit( $upload_dir[ 'basedir' ] ), '', $fullsizepath )), true, $fullsizepath ); if ( $result_code !== 200 ) { - if(!$this->get_attachment_if_exist($image->ID, $fullsizepath)){ // Save file to local from proxy. - $this->store_failed_attachment( $image->ID, 'images' ); + if(!Utility::sync_get_attachment_if_exist($image->ID, $fullsizepath)){ // Save file to local from proxy. + Utility::sync_store_failed_attachment( $image->ID, 'images' ); throw new \Exception(sprintf(__('Both local and remote files are missing. Unable to resize. (%s)', ud_get_stateless_media()->domain), $image->guid)); } } @@ -189,18 +158,18 @@ public function action_stateless_process_image() { @set_time_limit( -1 ); - // + // do_action( 'sm:pre::synced::image', $id); $metadata = wp_generate_attachment_metadata( $image->ID, $fullsizepath ); if(get_post_mime_type($image->ID) !== 'image/svg+xml'){ if ( is_wp_error( $metadata ) ) { - $this->store_failed_attachment( $image->ID, 'images' ); + Utility::sync_store_failed_attachment( $image->ID, 'images' ); throw new \Exception($metadata->get_error_message()); } - + if ( empty( $metadata ) ) { - $this->store_failed_attachment( $image->ID, 'images' ); + Utility::sync_store_failed_attachment( $image->ID, 'images' ); throw new \Exception(sprintf( __('No metadata generated for %1$s (ID %2$s).', ud_get_stateless_media()->domain), esc_html( get_the_title( $image->ID ) ), $image->ID)); } } @@ -208,8 +177,8 @@ public function action_stateless_process_image() { // If this fails, then it just means that nothing was changed (old value == new value) wp_update_attachment_metadata( $image->ID, $metadata ); - $this->store_current_progress( 'images', $id ); - $this->maybe_fix_failed_attachment( 'images', $image->ID ); + Utility::sync_store_current_progress( 'images', $id ); + Utility::sync_maybe_fix_failed_attachment( 'images', $image->ID ); do_action( 'sm:synced::image', $id, $metadata); return sprintf( __( '%1$s (ID %2$s) was successfully resized in %3$s seconds.', ud_get_stateless_media()->domain ), esc_html( get_the_title( $image->ID ) ), $image->ID, timer_stop() ); @@ -245,8 +214,8 @@ public function action_stateless_process_file() { $result_code = ud_get_stateless_media()->get_client()->get_media( str_replace( trailingslashit( $upload_dir[ 'basedir' ] ), '', $fullsizepath ), true, $fullsizepath ); if ( $result_code !== 200 ) { - if(!$this->get_attachment_if_exist($file->ID, $fullsizepath)){ // Save file to local from proxy. - $this->store_failed_attachment( $file->ID, 'other' ); + if(!Utility::sync_get_attachment_if_exist($file->ID, $fullsizepath)){ // Save file to local from proxy. + Utility::sync_store_failed_attachment( $file->ID, 'other' ); throw new \Exception(sprintf(__('File not found (%s)', ud_get_stateless_media()->domain), $file->guid)); } else{ @@ -268,7 +237,7 @@ public function action_stateless_process_file() { $metadata = wp_generate_attachment_metadata( $file->ID, $fullsizepath ); if ( is_wp_error( $metadata ) ) { - $this->store_failed_attachment( $file->ID, 'other' ); + Utility::sync_store_failed_attachment( $file->ID, 'other' ); throw new \Exception($metadata->get_error_message()); } @@ -285,8 +254,8 @@ public function action_stateless_process_file() { } - $this->store_current_progress( 'other', $id ); - $this->maybe_fix_failed_attachment( 'other', $file->ID ); + Utility::sync_store_current_progress( 'other', $id ); + Utility::sync_maybe_fix_failed_attachment( 'other', $file->ID ); return sprintf( __( '%1$s (ID %2$s) was successfully synchronised in %3$s seconds.', ud_get_stateless_media()->domain ), esc_html( get_the_title( $file->ID ) ), $file->ID, timer_stop() ); } @@ -302,7 +271,7 @@ public function action_stateless_process_non_library_file() { if(ud_get_stateless_media()->is_connected_to_gs() !== true){ throw new \Exception( __( 'Not connected to GCS', ud_get_stateless_media()->domain) ); } - + $upload_dir = wp_upload_dir(); $client = ud_get_stateless_media()->get_client(); @@ -342,7 +311,7 @@ public function action_get_images_media_ids() { $start_from = isset( $_REQUEST['start_from'] ) ? (int) $_REQUEST['start_from'] : 0; } - return $this->get_non_processed_media_ids( 'images', $images, $continue, $start_from ); + return Utility::sync_get_non_processed_media_ids( 'images', $images, $continue, $start_from ); } /** @@ -366,7 +335,7 @@ public function action_get_other_media_ids() { $start_from = isset( $_REQUEST['start_from'] ) ? (int) $_REQUEST['start_from'] : 0; } - return $this->get_non_processed_media_ids( 'other', $files, $continue, $start_from ); + return Utility::sync_get_non_processed_media_ids( 'other', $files, $continue, $start_from ); } /** @@ -390,8 +359,8 @@ public function action_get_non_library_files_id() { */ public function action_stateless_get_current_progresses() { return array( - 'images' => $this->retrieve_current_progress( 'images' ), - 'other' => $this->retrieve_current_progress( 'other' ), + 'images' => Utility::sync_retrieve_current_progress( 'images' ), + 'other' => Utility::sync_retrieve_current_progress( 'other' ), ); } @@ -400,25 +369,11 @@ public function action_stateless_get_current_progresses() { */ public function action_stateless_get_all_fails() { return array( - 'images' => $this->get_fails( 'images' ), - 'other' => $this->get_fails( 'other' ) + 'images' => Utility::sync_get_fails( 'images' ), + 'other' => Utility::sync_get_fails( 'other' ) ); } - /** - * Get_fails - * - * @param $mode - * @return mixed|void - */ - private function get_fails( $mode ) { - if ( $mode !== 'other' ) { - $mode = 'images'; - } - - return get_option( 'wp_stateless_failed_' . $mode ); - } - /** * Resets the current progress for a specific mode. */ @@ -428,143 +383,11 @@ public function action_stateless_reset_progress() { $mode = 'other'; } - $this->reset_current_progress( $mode ); + Utility::sync_reset_current_progress( $mode ); return true; } - /** - * Get_non_processed_media_ids - * - * @param $mode - * @param $files - * @param bool $continue - * @return array - * @throws \Exception - */ - private function get_non_processed_media_ids( $mode, $files, $continue = false, $start_from = 0 ) { - if(ud_get_stateless_media()->is_connected_to_gs() !== true){ - throw new \Exception( __( 'Not connected to GCS', ud_get_stateless_media()->domain) ); - } - - if ( $continue ) { - $progress = $this->retrieve_current_progress( $mode ); - - if ( false !== $progress ) { - if($start_from && $start_from != 0){ - // adding 1 because we subtracted 1 in js code for presentation. - $progress[1] = $start_from + 1; - } - $ids = array(); - foreach ( $files as $file ) { - $id = (int) $file->ID; - // only include IDs that have not been processed yet - if ( $id > $progress[0] || $id < $progress[1] ) { - $ids[] = $id; - } - } - return $ids; - } - } - - $this->reset_current_progress( $mode ); - - $ids = array(); - foreach ( $files as $file ) - $ids[] = (int)$file->ID; - - return $ids; - } - - /** - * @param $mode - * @param $id - */ - private function store_current_progress( $mode, $id ) { - if ( $mode !== 'other' ) { - $mode = 'images'; - } - - $first_processed = get_option( 'wp_stateless_' . $mode . '_first_processed' ); - if ( ! $first_processed ) { - update_option( 'wp_stateless_' . $mode . '_first_processed', $id ); - } - $last_processed = get_option( 'wp_stateless_' . $mode . '_last_processed' ); - if ( ! $last_processed || $id < (int) $last_processed ) { - update_option( 'wp_stateless_' . $mode . '_last_processed', $id ); - } - } - - /** - * @param $attachment_id - * @param $mode - */ - private function store_failed_attachment( $attachment_id, $mode ) { - if ( $mode !== 'other' ) { - $mode = 'images'; - } - - $fails = get_option( 'wp_stateless_failed_' . $mode ); - if ( !empty( $fails ) && is_array( $fails ) ) { - if ( !in_array( $attachment_id, $fails ) ) { - $fails[] = $attachment_id; - } - } else { - $fails = array( $attachment_id ); - } - - update_option( 'wp_stateless_failed_' . $mode, $fails ); - } - - /** - * @param $mode - * @param $attachment_id - */ - private function maybe_fix_failed_attachment( $mode, $attachment_id ) { - $fails = get_option( 'wp_stateless_failed_' . $mode ); - - if ( !empty( $fails ) && is_array( $fails ) ) { - if ( in_array( $attachment_id, $fails ) ) { - foreach (array_keys($fails, $attachment_id) as $key) { - unset($fails[$key]); - } - } - } - - update_option( 'wp_stateless_failed_' . $mode, $fails ); - } - - /** - * @param $mode - * @return array|bool - */ - private function retrieve_current_progress( $mode ) { - if ( $mode !== 'other' ) { - $mode = 'images'; - } - - $first_processed = get_option( 'wp_stateless_' . $mode . '_first_processed' ); - $last_processed = get_option( 'wp_stateless_' . $mode . '_last_processed' ); - - if ( ! $first_processed || ! $last_processed ) { - return false; - } - - return array( (int) $first_processed, (int) $last_processed ); - } - - /** - * @param $mode - */ - private function reset_current_progress( $mode ) { - if ( $mode !== 'other' ) { - $mode = 'images'; - } - - delete_option( 'wp_stateless_' . $mode . '_first_processed' ); - delete_option( 'wp_stateless_' . $mode . '_last_processed' ); - } - } } diff --git a/lib/classes/class-utility.php b/lib/classes/class-utility.php index e313a9718..95930a2a5 100644 --- a/lib/classes/class-utility.php +++ b/lib/classes/class-utility.php @@ -106,7 +106,7 @@ public static function randomize_filename( $filename ) { if(!empty($ext)){ $filename .= '.' . $ext; } - + return $filename; } @@ -156,7 +156,7 @@ public static function getCacheControl( $attachment_id = null, $metadata = array /** * Add/Update Media to Bucket * Fired for every action with image add or update - * + * * $force and $args params will no be passed on media library uploads. * This two will be passed on by compatibility. * @@ -195,7 +195,7 @@ public static function add_media( $metadata, $attachment_id, $force = false, $ar * * $force and $args params will no be passed on non media library uploads. * This two will be passed on by compatibility. - * + * * @since 2.2.4 * * @param bool $value This should return true if want to skip the sync. @@ -215,7 +215,7 @@ public static function add_media( $metadata, $attachment_id, $force = false, $ar $fullsizepath = wp_normalize_path( get_attached_file( $attachment_id ) ); $_cacheControl = self::getCacheControl( $attachment_id, $metadata, null ); $_contentDisposition = self::getContentDisposition( $attachment_id, $metadata, null ); - + // Ensure image upload to GCS when attachment is updated, // by checking if the attachment metadata is changed. if($attachment_id && !empty($metadata) && !$force){ @@ -265,9 +265,9 @@ public static function add_media( $metadata, $attachment_id, $force = false, $ar if(!empty($cloud_meta['filesize'])){ $metadata['filesize'] = $cloud_meta['filesize']; } - + /** - * + * */ $image_sizes = self::get_path_and_url($metadata, $attachment_id); foreach($image_sizes as $size => $img){ @@ -316,7 +316,7 @@ public static function add_media( $metadata, $attachment_id, $force = false, $ar if( !is_wp_error( $media ) ) { // @note We don't add storageClass because it's same as parent... $cloud_meta = self::generate_cloud_meta($cloud_meta, $media, $size, $img, $bucketLink); - + // Stateless mode: we don't need the local version. // Except when uploading the full size image first. if(self::can_delete_attachment($attachment_id, $args)){ @@ -346,18 +346,18 @@ public static function add_media( $metadata, $attachment_id, $force = false, $ar } /** - * Triggers when the media and it's thumbs are synced. - * - * $force and $args params will no be passed on non media library uploads. - * This two will be passed on by compatibility. - * - * @since 2.2.5 - * - * @param int $metadata Metadata for the attachment. - * @param string $attachment_id Attachment ID. - * @param bool $force (optional) Whether to force the sync even the file already exist in GCS. - * @param bool $args (optional) Whether to only sync the full size image. - */ + * Triggers when the media and it's thumbs are synced. + * + * $force and $args params will no be passed on non media library uploads. + * This two will be passed on by compatibility. + * + * @since 2.2.5 + * + * @param int $metadata Metadata for the attachment. + * @param string $attachment_id Attachment ID. + * @param bool $force (optional) Whether to force the sync even the file already exist in GCS. + * @param bool $args (optional) Whether to only sync the full size image. + */ do_action( 'wp_stateless_media_synced', $metadata, $attachment_id, $force, $args); } @@ -433,8 +433,8 @@ public static function get_path_and_url( $metadata, $attachment_id ){ // lets do nothing. } } - - + + $gs_name_path['__full'] = array( 'gs_name' => $gs_name, 'path' => $full_size_path, @@ -444,8 +444,8 @@ public static function get_path_and_url( $metadata, $attachment_id ){ 'width' => isset($metadata['width']) ? $metadata['width'] : null, 'height' => isset($metadata['height']) ? $metadata['height'] : null, ); - - + + /* Now we go through all available image sizes and upload them to Google Storage */ if( !empty( $metadata[ 'sizes' ] ) && is_array( $metadata[ 'sizes' ] ) ) { foreach( $metadata[ 'sizes' ] as $image_size => $data ) { @@ -502,62 +502,62 @@ public static function generate_cloud_meta( $cloud_meta, $media, $image_size, $i * * @param array $parts * @param boolean $encode - * @return string $url - */ + * @return string $url + */ public static function join_url( $parts, $encode=TRUE ){ if ( $encode ){ - if ( isset( $parts['user'] ) ) - $parts['user'] = rawurlencode( $parts['user'] ); - if ( isset( $parts['pass'] ) ) - $parts['pass'] = rawurlencode( $parts['pass'] ); - if ( isset( $parts['host'] ) && - !preg_match( '!^(\[[\da-f.:]+\]])|([\da-f.:]+)$!ui', $parts['host'] ) ) - $parts['host'] = rawurlencode( $parts['host'] ); - if ( !empty( $parts['path'] ) ) - $parts['path'] = preg_replace( '!%2F!ui', '/', - rawurlencode( $parts['path'] ) ); - if ( isset( $parts['query'] ) ) - $parts['query'] = rawurlencode( $parts['query'] ); - if ( isset( $parts['fragment'] ) ) - $parts['fragment'] = rawurlencode( $parts['fragment'] ); - } - + if ( isset( $parts['user'] ) ) + $parts['user'] = rawurlencode( $parts['user'] ); + if ( isset( $parts['pass'] ) ) + $parts['pass'] = rawurlencode( $parts['pass'] ); + if ( isset( $parts['host'] ) && + !preg_match( '!^(\[[\da-f.:]+\]])|([\da-f.:]+)$!ui', $parts['host'] ) ) + $parts['host'] = rawurlencode( $parts['host'] ); + if ( !empty( $parts['path'] ) ) + $parts['path'] = preg_replace( '!%2F!ui', '/', + rawurlencode( $parts['path'] ) ); + if ( isset( $parts['query'] ) ) + $parts['query'] = rawurlencode( $parts['query'] ); + if ( isset( $parts['fragment'] ) ) + $parts['fragment'] = rawurlencode( $parts['fragment'] ); + } + $url = ''; if ( !empty( $parts['scheme'] ) ) - $url .= $parts['scheme'] . ':'; + $url .= $parts['scheme'] . ':'; if ( isset( $parts['host'] ) ){ - $url .= '//'; - if ( isset( $parts['user'] ) ){ - $url .= $parts['user']; - if ( isset( $parts['pass'] ) ) - $url .= ':' . $parts['pass']; - $url .= '@'; - } - if ( preg_match( '!^[\da-f]*:[\da-f.:]+$!ui', $parts['host'] ) ) - $url .= '[' . $parts['host'] . ']'; // IPv6 - else - $url .= $parts['host']; // IPv4 or name - if ( isset( $parts['port'] ) ) - $url .= ':' . $parts['port']; - if ( !empty( $parts['path'] ) && $parts['path'][0] != '/' ) - $url .= '/'; + $url .= '//'; + if ( isset( $parts['user'] ) ){ + $url .= $parts['user']; + if ( isset( $parts['pass'] ) ) + $url .= ':' . $parts['pass']; + $url .= '@'; + } + if ( preg_match( '!^[\da-f]*:[\da-f.:]+$!ui', $parts['host'] ) ) + $url .= '[' . $parts['host'] . ']'; // IPv6 + else + $url .= $parts['host']; // IPv4 or name + if ( isset( $parts['port'] ) ) + $url .= ':' . $parts['port']; + if ( !empty( $parts['path'] ) && $parts['path'][0] != '/' ) + $url .= '/'; } if ( !empty( $parts['path'] ) ) - $url .= $parts['path']; + $url .= $parts['path']; if ( isset( $parts['query'] ) ) - $url .= '?' . $parts['query']; + $url .= '?' . $parts['query']; if ( isset( $parts['fragment'] ) ) - $url .= '#' . $parts['fragment']; + $url .= '#' . $parts['fragment']; return $url; } /** * add_webp_mime - * + * */ public function add_webp_mime($t, $user){ - $t['webp'] = 'image/webp'; - return $t; + $t['webp'] = 'image/webp'; + return $t; } /** @@ -584,15 +584,15 @@ public static function store_can_delete_attachment( $new_sizes, $image_meta, $at */ public static function can_delete_attachment($attachment_id, $args){ if( - ud_get_stateless_media()->get( 'sm.mode' ) === 'stateless' && + ud_get_stateless_media()->get( 'sm.mode' ) === 'stateless' && $args['no_thumb'] != true ){ // checks whether it's WP 5.3 and 'intermediate_image_sizes_advanced' is passed. // To be sure that we don't delete full size image before thumbnails are generated. if( wp_attachment_is_image($attachment_id) && - function_exists('is_wp_version_compatible') && - is_wp_version_compatible('5.3-RC4-46673') && + function_exists('is_wp_version_compatible') && + is_wp_version_compatible('5.3-RC4-46673') && !in_array($attachment_id, self::$can_delete_attachment) ){ return false; @@ -651,6 +651,192 @@ public static function isCallStackMatches( $callstack, $conditions ) { return false; } + + /** + * Fail over to image URL if not found on disk + * In case image not available on both local and bucket + * try to pull image from image URL in case it is accessible by some sort of proxy. + * + * @param: + * $url (int/string): URL of the image. + * $save_to (string): Path where to save the image. + * + * @return bool|int + * @throws \Exception + */ + public static function sync_get_attachment_if_exist($url, $save_to){ + if(is_int($url)) + $url = wp_get_attachment_url($url); + + $response = wp_remote_get( $url ); + if ( !is_wp_error($response) && is_array( $response ) ) { + if(!empty($response['response']['code']) && $response['response']['code'] == 200){ + try{ + if(wp_mkdir_p(dirname($save_to))){ + return file_put_contents($save_to, $response['body']); + } + } + catch(\Exception $e){ + throw $e; + } + } + } + return false; + } + + /** + * Store failed attachment + * @param $attachment_id + * @param $mode + */ + public static function sync_store_failed_attachment( $attachment_id, $mode ) { + if ( ! in_array( $mode, [ 'other', 'cli_images', 'cli_other' ] ) ) { + $mode = 'images'; + } + + $fails = get_option( 'wp_stateless_failed_' . $mode ); + if ( !empty( $fails ) && is_array( $fails ) ) { + if ( !in_array( $attachment_id, $fails ) ) { + $fails[] = $attachment_id; + } + } else { + $fails = array( $attachment_id ); + } + + update_option( 'wp_stateless_failed_' . $mode, $fails ); + } + + /** + * Checking maybe attachment have already fixed + * @param $mode + * @param $attachment_id + */ + public static function sync_maybe_fix_failed_attachment( $mode, $attachment_id ) { + $fails = get_option( 'wp_stateless_failed_' . $mode ); + + if ( !empty( $fails ) && is_array( $fails ) ) { + if ( in_array( $attachment_id, $fails ) ) { + foreach (array_keys($fails, $attachment_id) as $key) { + unset($fails[$key]); + } + } + } + + update_option( 'wp_stateless_failed_' . $mode, $fails ); + } + + /** + * Store current synchronization progress + * @param $mode + * @param $id + * @param $cli + */ + public static function sync_store_current_progress( $mode, $id, $cli = false ) { + if ( ! in_array( $mode, [ 'other', 'cli_images', 'cli_other' ] ) ) { + $mode = 'images'; + } + + $first_processed = get_option( 'wp_stateless_' . $mode . '_first_processed' ); + if ( ! $first_processed ) { + update_option( 'wp_stateless_' . $mode . '_first_processed', $id ); + } + $last_processed = get_option( 'wp_stateless_' . $mode . '_last_processed' ); + if ( ! $last_processed || $id < (int) $last_processed || $cli ) { + update_option( 'wp_stateless_' . $mode . '_last_processed', $id ); + } + } + + /** + * Get synchronization progress + * @param $mode + * @return array|bool + */ + public static function sync_retrieve_current_progress( $mode ) { + if ( ! in_array( $mode, [ 'other', 'cli_images', 'cli_other' ] ) ) { + $mode = 'images'; + } + + $first_processed = get_option( 'wp_stateless_' . $mode . '_first_processed' ); + $last_processed = get_option( 'wp_stateless_' . $mode . '_last_processed' ); + + if ( ! $first_processed || ! $last_processed ) { + return false; + } + + return array( (int) $first_processed, (int) $last_processed ); + } + + /** + * Reset synchronization progress + * @param $mode + */ + public static function sync_reset_current_progress( $mode ) { + if ( ! in_array( $mode, [ 'other', 'cli_images', 'cli_other' ] ) ) { + $mode = 'images'; + } + + delete_option( 'wp_stateless_' . $mode . '_first_processed' ); + delete_option( 'wp_stateless_' . $mode . '_last_processed' ); + } + + /** + * Get fails + * + * @param $mode + * @return mixed|void + */ + public static function sync_get_fails( $mode ) { + if ( ! in_array( $mode, [ 'other', 'cli_images', 'cli_other' ] ) ) { + $mode = 'images'; + } + + return get_option( 'wp_stateless_failed_' . $mode ); + } + + /** + * Get_non_processed_media_ids + * + * @param $mode + * @param $files + * @param bool $continue + * @param $start_from + * @return array + * @throws \Exception + */ + public static function sync_get_non_processed_media_ids( $mode, $files, $continue = false, $start_from = 0 ) { + if(ud_get_stateless_media()->is_connected_to_gs() !== true){ + throw new \Exception( __( 'Not connected to GCS', ud_get_stateless_media()->domain) ); + } + + if ( $continue ) { + $progress = self::sync_retrieve_current_progress( $mode ); + + if ( false !== $progress ) { + if($start_from && $start_from != 0){ + // adding 1 because we subtracted 1 in js code for presentation. + $progress[1] = $start_from + 1; + } + $ids = array(); + foreach ( $files as $file ) { + $id = (int) $file->ID; + // only include IDs that have not been processed yet + if ( $id > $progress[0] || $id < $progress[1] ) { + $ids[] = $id; + } + } + return $ids; + } + } + + self::sync_reset_current_progress( $mode ); + + $ids = array(); + foreach ( $files as $file ) + $ids[] = (int)$file->ID; + + return $ids; + } + } } } \ No newline at end of file diff --git a/lib/cli/class-sm-cli-command.php b/lib/cli/class-sm-cli-command.php index 11253c867..4c52aa30b 100644 --- a/lib/cli/class-sm-cli-command.php +++ b/lib/cli/class-sm-cli-command.php @@ -47,6 +47,119 @@ public function __construct( $args = array(), $assoc_args = array() ) { } + /** + * Sync Data + * + * ## OPTIONS + * + * + * : Which data we want to sync. May be images or files. + * + * --start + * : Indent (sql start). It's ignored on batches. + * + * --limit + * : Limit per query (sql limit) + * + * --end + * : Where ( on which row ) we should stop script. It's ignored on batches + * + * --batch + * : Number of Batch. Default is 1. + * + * --batches + * : General amount of batches. + * + * --b + * : Runs command using batches till it's done. Other parameters will be ignored. There are 10 batches by default. Batch is external command process + * + * --log + * : Show more information in command line + * + * --o + * : Process includes database optimization and transient removing. + * + * --force + * : Force sync all images + * + * --continue + * : Continue the last process + * + * --fix + * : Try to fix previously failed items + * + * --order + * : Order. May be ASC or DESC + * + * ## EXAMPLES + * + * wp stateless sync images --url=example.com --b + * : Run process looping 10 batches. Every batch is external command 'wp stateless sync images --url=example.com --batch= --batches=10' + * + * wp stateless sync images --url=example.com --b --batches=100 + * : Run process looping 100 batches. + * + * wp stateless sync images --url=example.com --b --batches=10 --batch=2 + * : Run second batch from 10 batches manually. + * + * wp stateless sync images --url=example.com --log + * : Run default process showing additional information in command line. + * + * wp stateless sync images --url=example.com --end=3000 --limit=50 + * : Run process from 1 to 3000 row. Splits process by limiting queries to 50 rows. So, the current example does 60 queries ( 3000 / 50 = 60 ) + * + * wp stateless sync images --url=example.com --start=777 --end=3000 --o + * : Run process from 777 to 3000 row. Also does database optimization and removes transient in the end. + * + * wp stateless sync images --force + * : Run process for all existing images. + * + * wp stateless sync images --continue + * : Run process from last synchronized image + * + * wp stateless sync files --fix + * : Run process and trying to fix previously failed items + * + * @synopsis [] [--start=] [--limit=] [--end=] [--batch=] [--batches=] [--b] [--log] [--o] [--force] [--continue] [--fix] [--order] + * @param $args + * @param $assoc_args + */ + public function sync( $args, $assoc_args ) { + + //** DB Optimization process */ + if( isset( $assoc_args[ 'o' ] ) ) { + $this->_before_command_run(); + } + //** Run batches */ + if( isset( $assoc_args[ 'b' ] ) ) { + if( empty( $args[0] ) ) { + WP_CLI::error( 'Invalid type parameter' ); + } + $this->_run_batches( 'sync', $args[0], $assoc_args ); + } + //** Or run command as is. */ + else { + if( !class_exists( 'SM_CLI_Sync' ) ) { + require_once( dirname( __FILE__ ) . '/class-sm-cli-sync.php' ); + } + if( class_exists( 'SM_CLI_Sync' ) ) { + $object = new SM_CLI_Sync( $args, $assoc_args ); + $controller = !empty( $args[0] ) ? $args[0] : false; + if( $controller && is_callable( array( $object, $controller ) ) ) { + call_user_func( array( $object, $controller ) ); + } else { + WP_CLI::error( 'Invalid type parameter' ); + } + } else { + WP_CLI::error( 'Class SM_CLI_Sync is undefined.' ); + } + } + //** Get rid of all transients and run DB optimization again */ + if( isset( $assoc_args[ 'o' ] ) ) { + $this->_after_command_run(); + } + } + /** * Upgrade Data * @@ -81,22 +194,22 @@ public function __construct( $args = array(), $assoc_args = array() ) { * * ## EXAMPLES * - * wp sm upgrade meta --url=example.com --b - * : Run process looping 10 batches. Every batch is external command 'wp sm upgrade meta --url=example.com --batch= --batches=10' + * wp stateless upgrade meta --url=example.com --b + * : Run process looping 10 batches. Every batch is external command 'wp stateless upgrade meta --url=example.com --batch= --batches=10' * - * wp sm upgrade meta --url=example.com --b --batches=100 + * wp stateless upgrade meta --url=example.com --b --batches=100 * : Run process looping 100 batches. * - * wp sm upgrade meta --url=example.com --b --batches=10 --batch=2 + * wp stateless upgrade meta --url=example.com --b --batches=10 --batch=2 * : Run second batch from 10 batches manually. * - * wp sm upgrade meta --url=example.com --log + * wp stateless upgrade meta --url=example.com --log * : Run default process showing additional information in command line. * - * wp sm upgrade meta --url=example.com --end=3000 --limit=50 + * wp stateless upgrade meta --url=example.com --end=3000 --limit=50 * : Run process from 1 to 3000 row. Splits process by limiting queries to 50 rows. So, the current example does 60 queries ( 3000 / 50 = 60 ) * - * wp sm upgrade meta --url=example.com --start=777 --end=3000 --o + * wp stateless upgrade meta --url=example.com --start=777 --end=3000 --o * : Run process from 777 to 3000 row. Also does database optimization and removes transient in the end. * * @synopsis [] [--start=] [--limit=] [--end=] [--batch=] [--batches=] [--b] [--log] [--o] @@ -154,9 +267,9 @@ private function _run_batches( $method, $type, $assoc_args ) { for( $i=1; $i<=$batches; $i++ ) { if( !empty( $this->url ) ) { - $command = "wp sm {$method} {$type} --batch={$i} --batches={$batches} --limit={$limit} --url={$this->url}"; + $command = "wp stateless {$method} {$type} --batch={$i} --batches={$batches} --limit={$limit} --url={$this->url}"; } else { - $command = "wp sm {$method} {$type} --batch={$i} --batches={$batches} --limit={$limit}"; + $command = "wp stateless {$method} {$type} --batch={$i} --batches={$batches} --limit={$limit}"; } WP_CLI::line( '...' ); @@ -226,6 +339,6 @@ private function _after_command_run(){ } /** Add the commands from above */ - WP_CLI::add_command( 'sm', 'SM_CLI_Command' ); + WP_CLI::add_command( 'stateless', 'SM_CLI_Command' ); } \ No newline at end of file diff --git a/lib/cli/class-sm-cli-sync.php b/lib/cli/class-sm-cli-sync.php new file mode 100644 index 000000000..97861b1aa --- /dev/null +++ b/lib/cli/class-sm-cli-sync.php @@ -0,0 +1,474 @@ +_prepare(); + + $timer = time(); + + $where = ''; + + if ( $this->fix ) { + $failed_attachments = Utility::sync_get_fails( 'cli_images' ); + + if ( !empty($failed_attachments) ) { + $where .= " AND ID IN (" . implode(',', $failed_attachments) . ") "; + } else { + WP_CLI::success( "Stateless Media is synced. No failed attachments." ); + exit; + } + } + + if ( $this->continue ) { + $current_progress = Utility::sync_retrieve_current_progress( 'cli_images' ); + + if ( isset($current_progress[1]) && !empty( $current_progress[1] )) { + $where .= " AND ID " . ( $this->order === 'DESC' ? ' < ' : ' > ' ) . $current_progress[1]; + } + } + + if ( ! $this->force ) { + $where .= " AND meta_id IS NULL " ; + } + + /** Get Total Amount of Attachments */ + $this->total = $wpdb->get_var(" + SELECT + COUNT(ID) + FROM {$wpdb->posts} + LEFT JOIN {$wpdb->postmeta} ON ID = post_id AND meta_key = 'sm_cloud' + WHERE post_type = 'attachment' AND post_mime_type LIKE 'image/%' " . $where . " + "); + + + if( $this->batch ) { + $this->output( "Running Batch {$this->batch} from {$this->batches}" ); + $range = round( $this->total / $this->batches ); + $this->start = ( $this->batch * $range ) - $range; + $this->end = $this->batches == $this->batch ? $this->total : $this->batch * $range; + $this->output( "Starting from {$this->start} row. " ); + $this->output( "And proceeding up to {$this->end} row." ); + } else { + $this->output( "Running in default way. Starting from {$this->start} row and proceeding up to end." ); + $this->end = $this->end ? $this->end : $this->total; + } + $media_to_proceed = $this->end - $this->start; + + //** Counters */ + $synced_images = 0; + + WP_CLI::line( 'Starting extract attachments.' ); + + for( $this->start; $this->start < $this->end; $this->start += $this->limit ) { + + $limit = ( $this->end - $this->start ) < $this->limit ? ( $this->end - $this->start ) : $this->limit; + + $this->output( 'Synced: ' . $synced_images . '. Extracting from ' . ( $this->start + 1 ) . ' to ' . ( $this->start + $limit ) ); + + /** + * Get Attachments data. + * + */ + $attachments = $wpdb->get_results( $wpdb->prepare( " + SELECT + ID + FROM {$wpdb->posts} + LEFT JOIN {$wpdb->postmeta} ON ID = post_id AND meta_key = 'sm_cloud' + WHERE post_type = 'attachment' AND post_mime_type LIKE 'image/%' " . $where . " ORDER BY ID " . $this->order . " + LIMIT %d, %d; + ", $this->start, $limit ), ARRAY_A ); + + if ( ! empty( $attachments ) ) { + foreach ($attachments as $i => $a) { + + try { + $response = $this->stateless_process_image($a['ID']); + if (!empty($response)) { + $synced_images++; + $this->output($response); + } + } catch( \Exception $e ) { + $this->output($e->getMessage()); + } + + /** Flush data */ + $wpdb->flush(); + @ob_flush(); + @flush(); + + } + + unset($attachments); + } + + } + WP_CLI::success( "Stateless Media is synced" ); + + WP_CLI::line( 'Media which have been checked: ' . number_format_i18n( $media_to_proceed ) ); + WP_CLI::line( 'Synced stateless for ' . number_format_i18n( $synced_images ) . ' attachments' ); + WP_CLI::line( 'Spent Time: ' . ( time() - $timer ) . ' sec' ); + + } + + + /** + * Sync files + * + */ + public function files() { + global $wpdb; + + /** Prepare arguments */ + $this->_prepare(); + + $timer = time(); + + $where = ''; + + if ( $this->fix ) { + $failed_attachments = Utility::sync_get_fails( 'cli_images' ); + + if ( !empty( $failed_attachments ) ) { + $where .= " AND ID IN (" . implode(',', $failed_attachments) . ") "; + } else { + WP_CLI::success( "Stateless Media is synced. No failed attachments." ); + exit; + } + } + + if ( $this->continue ) { + $current_progress = Utility::sync_retrieve_current_progress( 'cli_other' ); + + if ( isset( $current_progress[1] ) && !empty( $current_progress[1] )) { + $where .= " AND ID " . ( $this->order === 'DESC' ? ' < ' : ' > ' ) . $current_progress[1]; + } + } + + if ( ! $this->force ) { + $where .= " AND meta_id IS NULL " ; + } + + /** Get Total Amount of Attachments */ + $this->total = $wpdb->get_var(" + SELECT + COUNT(ID) + FROM {$wpdb->posts} + LEFT JOIN {$wpdb->postmeta} ON ID = post_id AND meta_key = 'sm_cloud' + WHERE post_type = 'attachment' AND post_mime_type NOT LIKE 'image/%' " . $where . " + "); + + + if( $this->batch ) { + $this->output( "Running Batch {$this->batch} from {$this->batches}" ); + $range = round( $this->total / $this->batches ); + $this->start = ( $this->batch * $range ) - $range; + $this->end = $this->batches == $this->batch ? $this->total : $this->batch * $range; + $this->output( "Starting from {$this->start} row. " ); + $this->output( "And proceeding up to {$this->end} row." ); + } else { + $this->output( "Running in default way. Starting from {$this->start} row and proceeding up to end." ); + $this->end = $this->end ? $this->end : $this->total; + } + $media_to_proceed = $this->end - $this->start; + + //** Counters */ + $synced_files = 0; + + WP_CLI::line( 'Starting extract attachments.' ); + + for( $this->start; $this->start < $this->end; $this->start += $this->limit ) { + + $limit = ( $this->end - $this->start ) < $this->limit ? ( $this->end - $this->start ) : $this->limit; + + $this->output( 'Synced: ' . $synced_files . '. Extracting from ' . ( $this->start + 1 ) . ' to ' . ( $this->start + $limit ) ); + + /** + * Get Attachments data. + * + */ + $attachments = $wpdb->get_results( $wpdb->prepare( " + SELECT + ID + FROM {$wpdb->posts} + LEFT JOIN {$wpdb->postmeta} ON ID = post_id AND meta_key = 'sm_cloud' + WHERE post_type = 'attachment' AND post_mime_type NOT LIKE 'image/%' " . $where . " ORDER BY ID " . $this->order . " + LIMIT %d, %d; + ", $this->start, $limit ), ARRAY_A ); + + if ( ! empty( $attachments ) ) { + foreach ($attachments as $i => $a) { + + try { + $response = $this->stateless_process_file($a['ID']); + if (!empty($response)) { + $synced_files++; + $this->output($response); + } + } catch( \Exception $e ) { + $this->output($e->getMessage()); + } + + /** Flush data */ + $wpdb->flush(); + @ob_flush(); + @flush(); + + } + + unset($attachments); + } + + } + WP_CLI::success( "Stateless Media is synced" ); + + WP_CLI::line( 'Media which have been checked: ' . number_format_i18n( $media_to_proceed ) ); + WP_CLI::line( 'Synced stateless for ' . number_format_i18n( $synced_files ) . ' attachments' ); + WP_CLI::line( 'Spent Time: ' . ( time() - $timer ) . ' sec' ); + + } + + + /** + * @param $id + * @return string + * @throws Exception + * Regenerate image sizes. + */ + public function stateless_process_image( $id ) { + + if(ud_get_stateless_media()->is_connected_to_gs() !== true){ + //WP_CLI::line( 'Starting extract attachments.' ); + throw new \Exception( __( 'Not connected to GCS', ud_get_stateless_media()->domain) ); + } + + @error_reporting( 0 ); + timer_start(); + + $image = get_post( $id ); + + if ( ! $image || 'attachment' != $image->post_type || 'image/' != substr( $image->post_mime_type, 0, 6 ) ) + throw new \Exception( sprintf( __( 'Failed resize: %s is an invalid image ID.', ud_get_stateless_media()->domain ), esc_html( $_REQUEST['id'] ) ) ); + + $fullsizepath = get_attached_file( $image->ID ); + + // If no file found + if ( false === $fullsizepath || ! file_exists( $fullsizepath ) ) { + $upload_dir = wp_upload_dir(); + + // Try get it and save + $result_code = ud_get_stateless_media()->get_client()->get_media( apply_filters( 'wp_stateless_file_name', str_replace( trailingslashit( $upload_dir[ 'basedir' ] ), '', $fullsizepath )), true, $fullsizepath ); + + if ( $result_code !== 200 ) { + if(!Utility::sync_get_attachment_if_exist($image->ID, $fullsizepath)){ // Save file to local from proxy. + Utility::sync_store_failed_attachment( $image->ID, 'cli_images' ); + throw new \Exception(sprintf(__('Both local and remote files are missing. Unable to resize. (%s)', ud_get_stateless_media()->domain), $image->guid)); + } + } + } + + @set_time_limit( -1 ); + + // + do_action( 'sm:pre::synced::image', $image->ID); + $metadata = wp_generate_attachment_metadata( $image->ID, $fullsizepath ); + + if(get_post_mime_type($image->ID) !== 'image/svg+xml'){ + if ( is_wp_error( $metadata ) ) { + Utility::sync_store_failed_attachment( $image->ID, 'cli_images' ); + throw new \Exception($metadata->get_error_message()); + } + + if ( empty( $metadata ) ) { + Utility::sync_store_failed_attachment( $image->ID, 'cli_images' ); + throw new \Exception(sprintf( __('No metadata generated for %1$s (ID %2$s).', ud_get_stateless_media()->domain), esc_html( get_the_title( $image->ID ) ), $image->ID)); + } + } + + // If this fails, then it just means that nothing was changed (old value == new value) + wp_update_attachment_metadata( $image->ID, $metadata ); + Utility::sync_store_current_progress( 'cli_images', $image->ID, true ); + Utility::sync_maybe_fix_failed_attachment( 'cli_images', $image->ID ); + do_action( 'sm:synced::image', $image->ID, $metadata); + + return sprintf( __( '%1$s (ID %2$s) was successfully resized in %3$s seconds.', ud_get_stateless_media()->domain ), esc_html( get_the_title( $image->ID ) ), $image->ID, timer_stop() ); + } + + /** + * @param $id + * @return string + * @throws \Exception + */ + public function stateless_process_file( $id ) { + + @error_reporting( 0 ); + timer_start(); + + if(ud_get_stateless_media()->is_connected_to_gs() !== true){ + throw new \Exception( __( 'Not connected to GCS', ud_get_stateless_media()->domain) ); + } + + $file = get_post( $id ); + + if ( ! $file || 'attachment' != $file->post_type ) + throw new \Exception( sprintf( __( 'Attachment not found: %s is an invalid file ID.', ud_get_stateless_media()->domain ), esc_html( $id ) ) ); + + $fullsizepath = get_attached_file( $file->ID ); + $local_file_exists = file_exists( $fullsizepath ); + + if ( false === $fullsizepath || ! $local_file_exists ) { + $upload_dir = wp_upload_dir(); + + // Try get it and save + $result_code = ud_get_stateless_media()->get_client()->get_media( str_replace( trailingslashit( $upload_dir[ 'basedir' ] ), '', $fullsizepath ), true, $fullsizepath ); + + if ( $result_code !== 200 ) { + if(!Utility::sync_get_attachment_if_exist($file->ID, $fullsizepath)){ // Save file to local from proxy. + Utility::sync_store_failed_attachment( $file->ID, 'cli_other' ); + throw new \Exception(sprintf(__('File not found (%s)', ud_get_stateless_media()->domain), $file->guid)); + } + else{ + $local_file_exists = true; + } + } + else{ + $local_file_exists = true; + } + } + + if($local_file_exists){ + $upload_dir = wp_upload_dir(); + + if ( !ud_get_stateless_media()->get_client()->media_exists( str_replace( trailingslashit( $upload_dir[ 'basedir' ] ), '', $fullsizepath ) ) ) { + + @set_time_limit( -1 ); + + $metadata = wp_generate_attachment_metadata( $file->ID, $fullsizepath ); + + if ( is_wp_error( $metadata ) ) { + Utility::sync_store_failed_attachment( $file->ID, 'cli_other' ); + throw new \Exception($metadata->get_error_message()); + } + + wp_update_attachment_metadata( $file->ID, $metadata ); + do_action( 'sm:synced::nonImage', $file->ID, $metadata); + + } + else{ + // Stateless mode: we don't need the local version. + if(ud_get_stateless_media()->get( 'sm.mode' ) === 'stateless'){ + unlink($fullsizepath); + } + } + + } + + Utility::sync_store_current_progress( 'cli_other', $file->ID, true ); + Utility::sync_maybe_fix_failed_attachment( 'cli_other', $file->ID ); + + return sprintf( __( '%1$s (ID %2$s) was successfully synchronised in %3$s seconds.', ud_get_stateless_media()->domain ), esc_html( get_the_title( $file->ID ) ), $file->ID, timer_stop() ); + } + + /** + * + */ + private function _prepare() { + $args = $this->assoc_args; + if( isset( $args[ 'b' ] ) ) { + WP_CLI::error( 'Invalid parameter --b. Command must not be run directly with --b parameter.' ); + } + $this->start = isset( $args[ 'start' ] ) && is_numeric( $args[ 'start' ] ) ? $args[ 'start' ] : 0; + $this->limit = isset( $args[ 'limit' ] ) && is_numeric( $args[ 'limit' ] ) ? $args[ 'limit' ] : 100; + $this->force = isset( $args[ 'force' ] ) ? true : false; + $this->continue = isset( $args[ 'continue' ] ) ? true : false; + $this->fix = isset( $args[ 'fix' ] ) ? true : false; + $this->order = isset( $args[ 'order' ]) && $args[ 'order' ] === 'ASC' ? 'ASC' : 'DESC'; + if( isset( $args[ 'batch' ] ) ) { + if( !is_numeric( $args[ 'batch' ] ) || $args[ 'batch' ] <= 0 ) { + WP_CLI::error( 'Invalid parameter --batch' ); + } + $this->batch = $args[ 'batch' ]; + $this->batches = isset( $args[ 'batches' ] ) ? $args[ 'batches' ] : 10; + if( !is_numeric( $args[ 'batches' ] ) || $args[ 'batches' ] <= 0 ) { + WP_CLI::error( 'Invalid parameter --batches' ); + } elseif ( $this->batch > $this->batches ) { + WP_CLI::error( '--batch parameter must is invalid. It must not equal or less then --batches' ); + } + } else { + $this->end = isset( $args[ 'end' ] ) && is_numeric( $args[ 'end' ] ) ? $args[ 'end' ] : false; + } + } + + +} \ No newline at end of file