From a45cfe6cafb5fff158198545fb5c601c3615503f Mon Sep 17 00:00:00 2001 From: Daniel Garcia Briseno Date: Mon, 15 Jul 2024 14:41:27 -0400 Subject: [PATCH 1/5] Add traceback to logException --- src/Helper/ErrorHandler.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Helper/ErrorHandler.php b/src/Helper/ErrorHandler.php index 6108649f..15196fcf 100644 --- a/src/Helper/ErrorHandler.php +++ b/src/Helper/ErrorHandler.php @@ -71,7 +71,7 @@ function handleError($msg, $errorCode=255) { } function logException(Exception $exception, string $prefix='') { - $message = $exception->getFile() . ":" . $exception->getLine() . " - " . $exception->getMessage(); + $message = $exception->getFile() . ":" . $exception->getLine() . " - " . $exception->getMessage() . "\n" . $exception->getTraceAsString(); logErrorMsg($message, $prefix); } From 20ee689dd58d9b3d1ff9709788c43d8ae746b41e Mon Sep 17 00:00:00 2001 From: Daniel Garcia Briseno Date: Mon, 15 Jul 2024 14:42:51 -0400 Subject: [PATCH 2/5] Selecting count(*) is faster than count(id) --- src/Database/ImgIndex.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Database/ImgIndex.php b/src/Database/ImgIndex.php index d2558504..66852444 100644 --- a/src/Database/ImgIndex.php +++ b/src/Database/ImgIndex.php @@ -54,12 +54,12 @@ public function insertScreenshot($date, $imageScale, $roi, $watermark, $layers, // old implementation removed for events strings // used to be $this->events->serialize(); - $old_events_layer_string = ""; + $old_events_layer_string = ""; // old if events labels are shown switch , removed for new implementation // used to be $this->eventsLabels; - $old_events_labels_bool = false; - + $old_events_labels_bool = false; + $sql = sprintf( "INSERT INTO screenshots " @@ -108,7 +108,7 @@ public function insertScreenshot($date, $imageScale, $roi, $watermark, $layers, try { $result = $this->_dbConnection->query($sql); } catch (Exception $e) { - throw new \Exception("Could not create screenshot in our database", 2, $e); + throw new \Exception("Could not create screenshot in our database", 2, $e); } return $this->_dbConnection->getInsertId(); @@ -516,7 +516,7 @@ protected function _getGroupForSourceId($sourceId) { * * @return array Array containing 1 next image and 1 prev date image */ - public function getClosestDataBeforeAndAfter($date, $sourceId) + public function getClosestDataBeforeAndAfter($date, $sourceId) { include_once HV_ROOT_DIR.'/../src/Helper/DateTimeConversions.php'; @@ -764,7 +764,7 @@ public function getDataCount($start, $end, $sourceId, $switchSources = false) { if($dataSplit){ $sql = sprintf( - "SELECT COUNT(id) as count FROM data WHERE (".$this->getDatasourceIDsString($sourceId)." AND date BETWEEN '%s' AND '%s') OR (".$this->getDatasourceIDsString($sourceId2)." AND date BETWEEN '%s' AND '%s') LIMIT 1;", + "SELECT COUNT(*) as count FROM data WHERE (".$this->getDatasourceIDsString($sourceId)." AND date BETWEEN '%s' AND '%s') OR (".$this->getDatasourceIDsString($sourceId2)." AND date BETWEEN '%s' AND '%s') LIMIT 1;", $this->_dbConnection->link->real_escape_string($startDate), $this->_dbConnection->link->real_escape_string($middleDate), $this->_dbConnection->link->real_escape_string($middleDate), @@ -772,7 +772,7 @@ public function getDataCount($start, $end, $sourceId, $switchSources = false) { ); }else{ $sql = sprintf( - "SELECT COUNT(id) as count FROM data WHERE ".$this->getDatasourceIDsString($sourceId)." AND date BETWEEN '%s' AND '%s' LIMIT 1;", + "SELECT COUNT(*) as count FROM data WHERE ".$this->getDatasourceIDsString($sourceId)." AND date BETWEEN '%s' AND '%s' LIMIT 1;", $this->_dbConnection->link->real_escape_string($startDate), $this->_dbConnection->link->real_escape_string($endDate) ); From a11bd156ab40e4561f958d76b2ad1676d1e576f1 Mon Sep 17 00:00:00 2001 From: Daniel Garcia Briseno Date: Tue, 16 Jul 2024 12:24:50 -0400 Subject: [PATCH 3/5] Update HelioviewerMovie added exceptions for functions that can't be called before the movie is processed. added tests --- src/Module/Movies.php | 13 +- src/Movie/HelioviewerMovie.php | 99 +++++++--- .../movies/HelioviewerMovieTest.php | 187 ++++++++++++++++++ 3 files changed, 270 insertions(+), 29 deletions(-) create mode 100644 tests/unit_tests/movies/HelioviewerMovieTest.php diff --git a/src/Module/Movies.php b/src/Module/Movies.php index 558b77c2..0e67bdf5 100644 --- a/src/Module/Movies.php +++ b/src/Module/Movies.php @@ -1294,10 +1294,15 @@ public function isYouTubeVideoExist($videoID) { } /** - * - * + * Checks if the given movie exists on disk. + * If the movie doesn't exist and allowRegeneration is true, then the movie + * will be queued for processing. */ - public function _verifyMediaExists($movie, $allowRegeneration=true) { + public function _verifyMediaExists(Movie_HelioviewerMovie $movie, $allowRegeneration=true): bool { + // If the movie isn't complete, it's guaranteed to not exist + if (!$movie->isComplete()) { + return false; + } // Check for missing movie or preview images $media_exists = true; @@ -1354,7 +1359,7 @@ public function playMovie() { // Check that the movie (in the requested format) as well as // its thumbnail images exist on disk. If not, silently // queue the movie for re-generation. - $this->_verifyMediaExists($movie, $allowRegeneration=true); + $this->_verifyMediaExists($movie, true); // Default options $defaults = array( diff --git a/src/Movie/HelioviewerMovie.php b/src/Movie/HelioviewerMovie.php index d4e2dc5f..9a54ac9a 100644 --- a/src/Movie/HelioviewerMovie.php +++ b/src/Movie/HelioviewerMovie.php @@ -37,6 +37,12 @@ use Helioviewer\Api\Event\EventsStateManager; +/** + * Exception to throw when performing an operation on a movie instance + * that hasn't been processed into a movie. + */ +class MovieNotCompletedException extends Exception {} + class Movie_HelioviewerMovie { const STATUS_QUEUED = 0; const STATUS_PROCESSING = 1; @@ -158,7 +164,7 @@ public function __construct($publicId, $format='mp4') { // ATTENTION! These two fields eventsLabels and eventSourceString needs to be kept in DB schema // We are keeping them to support old takeScreenshot , queueMovie requests - + // Events Manager $events_state_from_info = json_decode($info['eventsState'], true); @@ -170,9 +176,6 @@ public function __construct($publicId, $format='mp4') { // Regon of interest $this->_roi = Helper_RegionOfInterest::parsePolygonString($info['roi'], $info['imageScale']); - - // Get timestamps for frames in the key movie layer - $this->_getTimeStamps(); } private function _dbSetup() { @@ -268,9 +271,13 @@ public function build() { /** * Returns information about the completed movie * + * @throws * @return array A list of movie properties and a URL to the finished movie */ public function getCompletedMovieInformation($verbose=false) { + if (!$this->isComplete()) { + throw new MovieNotCompletedException("Movie $this->publicId has not been completed."); + } $info = array( 'frameRate' => $this->frameRate, @@ -288,7 +295,7 @@ public function getCompletedMovieInformation($verbose=false) { if ($verbose) { $extra = array( 'timestamp' => $this->timestamp, - 'duration' => $this->getDuration(), + 'duration' => $this->_getDuration(), 'imageScale' => $this->imageScale, 'layers' => $this->_layers->serialize(), 'events' => $this->_eventsManager->getState(), @@ -303,6 +310,10 @@ public function getCompletedMovieInformation($verbose=false) { return $info; } + public function isComplete(): bool { + return $this->status == Movie_HelioviewerMovie::STATUS_COMPLETED; + } + /** * Returns an array of filepaths to the movie's preview images */ @@ -325,10 +336,21 @@ public function getFilepath($highQuality=false) { return $this->_buildDir().$this->_buildFilename($highQuality); } - public function getDuration() { + private function _getDuration() { return $this->numFrames / $this->frameRate; } + /** + * Returns the duration of the movie in seconds + * @throws MovieNotCreatedException if the movie has not been processed + */ + public function getDuration() { + if (!$this->isComplete()) { + throw new MovieNotCompletedException("Duration for $this->publicId is unknown since the movie has not been processed yet."); + } + return $this->_getDuration(); + } + public function getURL() { return str_replace(HV_CACHE_DIR, HV_CACHE_URL, $this->_buildDir()) . $this->_buildFilename(); } @@ -397,6 +419,20 @@ private function _buildDir() { $this->publicId); } + private function _getStartDate() { + if (is_null($this->startDate)) { + $this->_prepDates(); + } + return $this->startDate; + } + + private function _getEndDate() { + if (is_null($this->endDate)) { + $this->_prepDates(); + } + return $this->endDate; + } + /** * Determines filename to use for the movie * @@ -405,11 +441,8 @@ private function _buildDir() { * @return string Movie filename */ private function _buildFilename($highQuality=false) { - if (is_null($this->startDate)) { - $this->_prepDates(); - } - $start = str_replace(array(':', '-', ' '), '_', $this->startDate); - $end = str_replace(array(':', '-', ' '), '_', $this->endDate); + $start = str_replace(array(':', '-', ' '), '_', $this->_getStartDate()); + $end = str_replace(array(':', '-', ' '), '_', $this->_getEndDate()); $suffix = ($highQuality && $this->format == 'mp4') ? '-hq' : ''; @@ -440,7 +473,7 @@ private function _buildMovieFrames($watermark) { 'movie' => true, 'size' => $this->size, 'followViewport' => $this->followViewport, - 'startDate' => $this->startDate, + 'startDate' => $this->_getStartDate(), 'reqStartDate' => $this->reqStartDate, 'reqEndDate' => $this->reqEndDate, 'reqObservationDate' => $this->reqObservationDate, @@ -454,7 +487,7 @@ private function _buildMovieFrames($watermark) { $numFailures = 0; // Compile frames - foreach ($this->_timestamps as $time) { + foreach ($this->_getTimeStamps() as $time) { $filepath = sprintf('%sframes/frame%d.bmp', $this->directory, $frameNum); @@ -546,6 +579,11 @@ private function _createPreviewImages(&$screenshot) { $preview->destroy(); } + private function markFinished(string $format, float $time_to_build) { + $this->_db->markMovieAsFinished($this->id, $format, $time_to_build); + $this->status = Movie_HelioviewerMovie::STATUS_COMPLETED; + } + /** * Builds the requested movie * @@ -630,7 +668,7 @@ private function _encodeMovie() { // Mark mp4 movie as completed $t2 = time(); - $this->_db->markMovieAsFinished($this->id, 'mp4', $t2 - $t1); + $this->markFinished('mp4', $t2 - $t1); // Create a low-quality webm movie for in-browser use if requested @@ -640,13 +678,18 @@ private function _encodeMovie() { // Mark movie as completed $t4 = time(); - $this->_db->markMovieAsFinished($this->id, 'webm', $t4 - $t3); + $this->markFinished('webm', $t4 - $t3); } /** * Returns a human-readable title for the video + * @throws MovieNotCompletedException since the title relies on the layer dates from processing the movie. */ public function getTitle() { + if (!$this->isComplete()) { + throw new MovieNotCompletedException("The title for $this->publicId is not available yet"); + } + date_default_timezone_set('UTC'); $layerString = $this->_layers->toHumanReadableString(); @@ -661,14 +704,14 @@ public function getTitle() { public function getDateString() { date_default_timezone_set('UTC'); - if (substr($this->startDate, 0, 9) == substr($this->endDate, 0, 9)) { - $endDate = substr($this->endDate, 11); + if (substr($this->_getStartDate(), 0, 9) == substr($this->_getEndDate(), 0, 9)) { + $endDate = substr($this->_getEndDate(), 11); } else { - $endDate = $this->endDate; + $endDate = $this->_getEndDate(); } - return sprintf('%s - %s UTC', $this->startDate, $endDate); + return sprintf('%s - %s UTC', $this->_getStartDate(), $endDate); } /** @@ -680,7 +723,12 @@ public function getDateString() { * included may be reduced to ensure that the total number of * SubFieldImages needed does not exceed HV_MAX_MOVIE_FRAMES */ - private function _getTimeStamps() { + private function _getTimeStamps(): array { + // If timestamps have already been processed, return them. + if (!empty($this->_timestamps)) { + return $this->_timestamps; + } + $this->_dbSetup(); $layerCounts = array(); @@ -721,6 +769,8 @@ private function _getTimeStamps() { $index = round($i * (sizeOf($entireRange) / $numFrames)); array_push($this->_timestamps, $entireRange[$index]['date']); } + + return $this->_timestamps; } /** @@ -746,8 +796,8 @@ private function _setMovieDimensions() { private function _prepDates() { if ($this->status != 2) { // Store actual start and end dates that will be used for the movie - $this->startDate = $this->_timestamps[0]; - $this->endDate = $this->_timestamps[sizeOf($this->_timestamps) - 1]; + $this->startDate = $this->_getTimeStamps()[0]; + $this->endDate = $this->_getTimeStamps()[sizeOf($this->_getTimeStamps()) - 1]; } } @@ -762,7 +812,7 @@ private function _setMovieProperties() { $this->filename = $this->_buildFilename(); - $this->numFrames = sizeOf($this->_timestamps); + $this->numFrames = sizeOf($this->_getTimeStamps()); if ($this->numFrames == 0) { $this->_abort('No images available for the requested time range'); @@ -796,7 +846,7 @@ private function _setMovieProperties() { // Update movie entry in database with new details $this->_db->storeMovieProperties( - $this->id, $this->startDate, $this->endDate, $this->numFrames, + $this->id, $this->_getStartDate(), $this->_getEndDate(), $this->numFrames, $this->frameRate, $this->movieLength, $width, $height ); } @@ -869,6 +919,5 @@ public function getMoviePlayerHTML() { diff --git a/tests/unit_tests/movies/HelioviewerMovieTest.php b/tests/unit_tests/movies/HelioviewerMovieTest.php new file mode 100644 index 00000000..feed1560 --- /dev/null +++ b/tests/unit_tests/movies/HelioviewerMovieTest.php @@ -0,0 +1,187 @@ + + */ + +use PHPUnit\Framework\TestCase; +use Helioviewer\Api\Event\EventsStateManager; + +// Dependencies +include_once HV_ROOT_DIR.'/../src/Helper/RegionOfInterest.php'; +include_once HV_ROOT_DIR.'/../src/Database/MovieDatabase.php'; +include_once HV_ROOT_DIR.'/../src/Movie/HelioviewerMovie.php'; + +final class HelioviewerMovieTest extends TestCase +{ + const TEST_LAYER = "[SOHO,LASCO,C2,white-light,2,100,0,60,1,2024-07-11T09:03:05.000Z]"; + + /** + * Inserts a test movie into the database and returns its public ID + */ + static private function InsertSohoTestMovie(): string { + // Get a handle to the movie database + $movieDb = new Database_MovieDatabase(); + // Set up arguments needed to insert a movie into the database + $imageScale = 16; + $roi = new Helper_RegionOfInterest(-16600, -8000, 16600, 8000, $imageScale); + $soho_layer = HelioviewerMovieTest::TEST_LAYER; + $layers = new Helper_HelioviewerLayers($soho_layer); + $events_manager = EventsStateManager::buildFromEventsState([]); + // Insert a test movie + $dbId = $movieDb->insertMovie( + "2023-12-01 00:00:00", + "2023-12-01 01:00:00", + "2023-12-01 00:30:00", + $imageScale, + $roi->getPolygonString(), + 10, + true, + "[SOHO,LASCO,C2,white-light,2,100,0,60,1,2024-07-11T09:03:05.000Z]", + bindec($layers->getBitMask()), + $events_manager->export(), + false, + false, + false, + "disabled", + 0, + 0, + $layers->length(), + 1, + 1, + 5, + 0, + false, + [ + "labels" => "", + "trajectories" => "" + ] + ); + + $movieDb->insertMovieFormat($dbId, 'mp4'); + // Return the database ID as a public ID. + return alphaID($dbId, false, 5, HV_MOVIE_ID_PASS); + } + + public function testBuildMovie() { + $movie_id = $this->InsertSohoTestMovie(); + $movie_builder = new Movie_HelioviewerMovie($movie_id); + // When movie is first inserted, it should have a queued status + $this->assertEquals(Movie_HelioviewerMovie::STATUS_QUEUED, $movie_builder->status); + // Build the movie + $movie_builder->build(); + // After the movie is built, status should be completed. + $this->assertEquals(Movie_HelioviewerMovie::STATUS_COMPLETED, $movie_builder->status); + + // After reloading the movie, status should still be completed. + $movie_builder = new Movie_HelioviewerMovie($movie_id); + $this->assertEquals(Movie_HelioviewerMovie::STATUS_COMPLETED, $movie_builder->status); + + return $movie_id; + } + + /** + * @depends testBuildMovie + */ + public function testIsComplete(string $movie_id) { + $completed = new Movie_HelioviewerMovie($movie_id); + $this->assertTrue($completed->isComplete()); + $not_completed_id = $this->InsertSohoTestMovie(); + $not_completed = new Movie_HelioviewerMovie($not_completed_id); + $this->assertFalse($not_completed->isComplete()); + } + + /** + * @depends testBuildMovie + */ + public function testGetCompletedMovieInformation(string $movie_id) { + $movie = new Movie_HelioviewerMovie($movie_id); + // These should align with the generated test movie. + // See InsertSohoTestMovie + $checkNonVerboseInfo = function($info, $movie_id) { + $this->assertEquals(1, $info['frameRate']); + $this->assertEquals(5, $info['numFrames']); + // These dates line up with the default environment's test data. + $this->assertEquals("2023-12-01 00:00:07", $info['startDate']); + $this->assertEquals("2023-12-01 00:48:07", $info['endDate']); + $this->assertEquals(2076, $info['width']); + $this->assertEquals(1000, $info['height']); + $this->assertEquals("SOHO LASCO C2 white-light (2023-12-01 00:00:07 - 00:48:07 UTC)", $info['title']); + $thumbnails = [ + 'icon' => HV_CACHE_URL . "/movies/2024/07/16/$movie_id/preview-icon.png", + 'small' => HV_CACHE_URL . "/movies/2024/07/16/$movie_id/preview-small.png", + 'medium' => HV_CACHE_URL . "/movies/2024/07/16/$movie_id/preview-medium.png", + 'large' => HV_CACHE_URL . "/movies/2024/07/16/$movie_id/preview-large.png", + 'full' => HV_CACHE_URL . "/movies/2024/07/16/$movie_id/preview-full.png", + ]; + $this->assertEquals($thumbnails, $info['thumbnails']); + $url = HV_CACHE_URL . "/movies/2024/07/16/$movie_id/2023_12_01_00_00_07_2023_12_01_00_48_07_LASCO_C2.mp4"; + $this->assertEquals($url, $info['url']); + }; + $info = $movie->getCompletedMovieInformation(); + $checkNonVerboseInfo($info, $movie_id); + + // Test verbose info + $verboseInfo = $movie->getCompletedMovieInformation(true); + $checkNonVerboseInfo($verboseInfo, $movie_id); + // This should contain the time the movie was created + $timestamp = DateTime::createFromFormat("Y-m-d H:i:s", $verboseInfo['timestamp']); + $today = new DateTime(); + // At least compare they have the same date + $this->assertEquals($today->format('Y-m-d'), $timestamp->format('Y-m-d')); + $this->assertEquals(5.0, $verboseInfo['duration']); + $this->assertEquals(16, $verboseInfo['imageScale']); + $this->assertEquals(HelioviewerMovieTest::TEST_LAYER, $verboseInfo['layers']); + $this->assertIsArray($verboseInfo['events']); + $this->assertEquals(-16600, $verboseInfo['x1']); + $this->assertEquals(-8000, $verboseInfo['y1']); + $this->assertEquals(16600, $verboseInfo['x2']); + $this->assertEquals(8000, $verboseInfo['y2']); + + // Test case where the movie has not been built + $new_id = $this->InsertSohoTestMovie(); + $movie = new Movie_HelioviewerMovie($new_id); + $this->expectException(MovieNotCompletedException::class); + $this->expectExceptionMessage($new_id); + $info = $movie->getCompletedMovieInformation(); + } + + public function testFilePath() { + $movie_id = $this->InsertSohoTestMovie(); + $movie = new Movie_HelioviewerMovie($movie_id); + $this->assertStringStartsWith(HV_CACHE_DIR . "/movies", $movie->getFilepath()); + $this->assertStringContainsString($movie_id, $movie->getFilepath()); + } + + /** + * @depends testBuildMovie + */ + public function testGetDuration(string $movie_id) { + // Check the duration of the processed movie. + $completed_movie = new Movie_HelioviewerMovie($movie_id); + $this->assertEquals(5.0, $completed_movie->getDuration()); + + // New movies that haven't been processed don't have a duration. + $movie_id = $this->InsertSohoTestMovie(); + $new_movie = new Movie_HelioviewerMovie($movie_id); + $this->expectException(MovieNotCompletedException::class); + $this->expectExceptionMessage($movie_id); + $new_movie->getDuration(); + } + + /** + * @depends testBuildMovie + */ + public function testGetTitle(string $movie_id) { + // Test the expected title for the test movie + $completed = new Movie_HelioviewerMovie($movie_id); + $this->assertEquals("SOHO LASCO C2 white-light (2023-12-01 00:00:07 - 00:48:07 UTC)", $completed->getTitle()); + + // New movies title is only available after processing, should throw an exception + // if you attempt to get the title before its ready. + $movie_id = $this->InsertSohoTestMovie(); + $new_movie = new Movie_HelioviewerMovie($movie_id); + $this->expectException(MovieNotCompletedException::class); + $this->expectExceptionMessage($movie_id); + $new_movie->getTitle(); + } +} From 5d8be34c0522c28423869c2fc12494efa51898db Mon Sep 17 00:00:00 2001 From: Daniel Garcia Briseno Date: Tue, 16 Jul 2024 13:17:11 -0400 Subject: [PATCH 4/5] Use TEST_LAYER --- tests/unit_tests/movies/HelioviewerMovieTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit_tests/movies/HelioviewerMovieTest.php b/tests/unit_tests/movies/HelioviewerMovieTest.php index feed1560..ffc99f76 100644 --- a/tests/unit_tests/movies/HelioviewerMovieTest.php +++ b/tests/unit_tests/movies/HelioviewerMovieTest.php @@ -36,7 +36,7 @@ static private function InsertSohoTestMovie(): string { $roi->getPolygonString(), 10, true, - "[SOHO,LASCO,C2,white-light,2,100,0,60,1,2024-07-11T09:03:05.000Z]", + HelioviewerMovieTest::TEST_LAYER, bindec($layers->getBitMask()), $events_manager->export(), false, From 0897f2675c0327b5a83f725f634625b65867eaf2 Mon Sep 17 00:00:00 2001 From: Daniel Garcia Briseno Date: Tue, 16 Jul 2024 13:22:29 -0400 Subject: [PATCH 5/5] Throw an exception in _getTimeStamps on failure --- src/Movie/HelioviewerMovie.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Movie/HelioviewerMovie.php b/src/Movie/HelioviewerMovie.php index 9a54ac9a..2e3b5647 100644 --- a/src/Movie/HelioviewerMovie.php +++ b/src/Movie/HelioviewerMovie.php @@ -42,6 +42,7 @@ * that hasn't been processed into a movie. */ class MovieNotCompletedException extends Exception {} +class MovieLookupException extends Exception {} class Movie_HelioviewerMovie { const STATUS_QUEUED = 0; @@ -737,6 +738,9 @@ private function _getTimeStamps(): array { // duration for each layer foreach ($this->_layers->toArray() as $layer) { $n = $this->_db->getDataCount($this->reqStartDate, $this->reqEndDate, $layer['sourceId'], $this->switchSources); + if ($n === false) { + throw new MovieLookupException("Failed to query data count for $this->publicId on source " . $layer['sourceId']); + } $layerCounts[$layer['sourceId']] = $n; }