-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Issue 47144: Handle large files on Panorama Public via Symlinks (#344)
* - PanoramaPublicFileImporter logs to the job log, and throws an exception if any of the datafileurls could not be fixed. - PanoramaPublicSymlinkManager.moveAndSymLinkDirectory takes a Logger parameter so the log output can go to the job log - Added a PanoramaPublicMetadataImporter. I moved some of the code out of CopyExperimentFinalTask into this class. This creates a row in the panoramapublic.experimentannotations table. It runs before PanoramaPublicFileImporter so that if there is an error, e.g. datafileurls cannot be fixed, the container can be deleted to move files back to the source container. - Updated test - import a document into a subfolder of the container file root. * - Fire symlink update events only when file / container being moved / renamed / deleted is in the Panorama Public project. We don't expect folders in other projects to contain symlink targets. - When handling folder rename (ContainerListener.propertyChange), pass the full paths of the old and renamed containers instead of just the folder names. Otherwise, it can lead to updating all symlinks that have the old folder name in the path. - When deleting a folder, use ExperimentAnnotationsManager.getExperimentIncludesContainer(c) to lookup the experiment. This method will return the experiment that contains runs from the folder even if it is a subfolder of the folder where the experiment was created. - When an experiment folder in Panorama Public is deleted, move the files back to next highest experiment version if one exists. Otherwise, move the files back to the source folder. * Rework datafile alignment * Scope datafile url to correct container * Removed PanoramaPublicFileWriter. * Limit the number of containers to look at when updating symlinks. This should only include the source container in the submitter's project as well as any containers with older versions of the data on Panorama Public. * Remove code to lookup runs in the source container when aligning datafileUrls. This should not be required anymore due to LabKey/targetedms#724. Set filePathRoot on the copied expRun to be the target container's file root. Log error if the data file path is unexpected, i.e. it does not contain "Run<runid>" Co-authored-by: vagisha <vagisha@gmail.com> Co-authored-by: Josh Eckels <jeckels@labkey.com> Co-authored-by: labkey-sweta <swetaj@labkey.com>
- Loading branch information
4 people
authored
Jul 1, 2023
1 parent
8fe5720
commit 5bcf018
Showing
20 changed files
with
1,722 additions
and
314 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
171 changes: 171 additions & 0 deletions
171
panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicFileImporter.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,171 @@ | ||
| package org.labkey.panoramapublic; | ||
|
|
||
| import org.apache.logging.log4j.Logger; | ||
| import org.jetbrains.annotations.Nullable; | ||
| import org.labkey.api.admin.AbstractFolderImportFactory; | ||
| import org.labkey.api.admin.FolderImportContext; | ||
| import org.labkey.api.admin.FolderImporter; | ||
| import org.labkey.api.admin.ImportException; | ||
| import org.labkey.api.admin.SubfolderWriter; | ||
| import org.labkey.api.data.Container; | ||
| import org.labkey.api.exp.api.ExpData; | ||
| import org.labkey.api.exp.api.ExpRun; | ||
| import org.labkey.api.exp.api.ExperimentService; | ||
| import org.labkey.api.files.FileContentService; | ||
| import org.labkey.api.pipeline.PipelineJob; | ||
| import org.labkey.api.pipeline.PipelineService; | ||
| import org.labkey.api.query.BatchValidationException; | ||
| import org.labkey.api.security.User; | ||
| import org.labkey.api.writer.VirtualFile; | ||
| import org.labkey.panoramapublic.pipeline.CopyExperimentPipelineJob; | ||
|
|
||
| import java.io.File; | ||
| import java.nio.file.Files; | ||
| import java.nio.file.Path; | ||
| import java.nio.file.Paths; | ||
| import java.util.List; | ||
| import java.util.Objects; | ||
|
|
||
| /** | ||
| * This importer does a file move instead of copy to the temp directory and creates a symlink in place of the original | ||
| * file. | ||
| */ | ||
| public class PanoramaPublicFileImporter implements FolderImporter | ||
| { | ||
| @Override | ||
| public String getDataType() | ||
| { | ||
| return PanoramaPublicManager.PANORAMA_PUBLIC_FILES; | ||
| } | ||
|
|
||
| @Override | ||
| public String getDescription() | ||
| { | ||
| return "Panorama Public Files"; | ||
| } | ||
|
|
||
| @Override | ||
| public void process(@Nullable PipelineJob job, FolderImportContext ctx, VirtualFile root) throws Exception | ||
| { | ||
| Logger log = ctx.getLogger(); | ||
|
|
||
| FileContentService fcs = FileContentService.get(); | ||
| if (null == fcs) | ||
| return; | ||
|
|
||
| File targetRoot = fcs.getFileRoot(ctx.getContainer()); | ||
|
|
||
| if (null == targetRoot) | ||
| { | ||
| log.error("File copy target folder not found: " + ctx.getContainer().getPath()); | ||
| return; | ||
| } | ||
|
|
||
| if (null == job) | ||
| { | ||
| log.error("Pipeline job not found."); | ||
| return; | ||
| } | ||
|
|
||
| if (job instanceof CopyExperimentPipelineJob expJob) | ||
| { | ||
| File targetFiles = new File(targetRoot.getPath(), FileContentService.FILES_LINK); | ||
|
|
||
| // Get source files including resolving subfolders | ||
| String divider = FileContentService.FILES_LINK + File.separator + PipelineService.EXPORT_DIR; | ||
| String subProject = root.getLocation().substring(root.getLocation().lastIndexOf(divider) + divider.length()); | ||
| subProject = subProject.replace(File.separator + SubfolderWriter.DIRECTORY_NAME, ""); | ||
|
|
||
| Path sourcePath = Paths.get(fcs.getFileRoot(expJob.getExportSourceContainer()).getPath(), subProject); | ||
| File sourceFiles = Paths.get(sourcePath.toString(), FileContentService.FILES_LINK).toFile(); | ||
|
|
||
| if (!targetFiles.exists()) | ||
| { | ||
| log.warn("Panorama public file copy target not found. Creating directory: " + targetFiles); | ||
| Files.createDirectories(targetFiles.toPath()); | ||
| } | ||
|
|
||
| log.info("Moving files and creating sym links in folder " + ctx.getContainer().getPath()); | ||
| PanoramaPublicSymlinkManager.get().moveAndSymLinkDirectory(expJob, sourceFiles, targetFiles, false, log); | ||
|
|
||
| alignDataFileUrls(expJob.getUser(), ctx.getContainer(), log); | ||
| } | ||
| } | ||
|
|
||
| private void alignDataFileUrls(User user, Container targetContainer, Logger log) throws BatchValidationException, ImportException | ||
| { | ||
| log.info("Aligning data files urls in folder: " + targetContainer.getPath()); | ||
|
|
||
| FileContentService fcs = FileContentService.get(); | ||
| if (null == fcs) | ||
| return; | ||
|
|
||
| ExperimentService expService = ExperimentService.get(); | ||
| List<? extends ExpRun> runs = expService.getExpRuns(targetContainer, null, null); | ||
| boolean errors = false; | ||
|
|
||
| Path fileRootPath = fcs.getFileRootPath(targetContainer, FileContentService.ContentType.files); | ||
| if(fileRootPath == null || !Files.exists(fileRootPath)) | ||
| { | ||
| throw new ImportException("File root path for container " + targetContainer.getPath() + " does not exist: " + fileRootPath); | ||
| } | ||
|
|
||
| for (ExpRun run : runs) | ||
| { | ||
| run.setFilePathRootPath(fileRootPath); | ||
| run.save(user); | ||
| log.debug("Setting filePathRoot on copied run: " + run.getName() + " to: " + fileRootPath); | ||
|
|
||
| for (ExpData data : run.getAllDataUsedByRun()) | ||
| { | ||
| if (null != data.getRun() && data.getDataFileUrl().contains(FileContentService.FILES_LINK)) | ||
| { | ||
| String[] parts = Objects.requireNonNull(data.getFilePath()).toString().split("Run\\d+"); | ||
|
|
||
| if (parts.length > 1) | ||
| { | ||
| String fileName = parts[1]; | ||
| Path newDataPath = Paths.get(fileRootPath.toString(), fileName); | ||
|
|
||
| if (newDataPath.toFile().exists()) | ||
| { | ||
| data.setDataFileURI(newDataPath.toUri()); | ||
| data.save(user); | ||
| log.debug("Setting dataFileUri on copied data: " + data.getName() + " to: " + newDataPath); | ||
| } | ||
| else | ||
| { | ||
| log.error("Data file not found: " + newDataPath.toUri()); | ||
| errors = true; | ||
| } | ||
| } | ||
| else | ||
| { | ||
| log.error("Unexpected data file path. Could not align dataFileUri. " + data.getFilePath().toString()); | ||
| errors = true; | ||
| } | ||
| } | ||
| } | ||
| } | ||
| if (errors) | ||
| { | ||
| throw new ImportException("Data files urls could not be aligned."); | ||
| } | ||
| } | ||
|
|
||
| public static class Factory extends AbstractFolderImportFactory | ||
| { | ||
| @Override | ||
| public FolderImporter create() | ||
| { | ||
| return new PanoramaPublicFileImporter(); | ||
| } | ||
|
|
||
| @Override | ||
| public int getPriority() | ||
| { | ||
| // We want this to run last to do exp.data.datafileurl cleanup | ||
| return PanoramaPublicManager.PRIORITY_PANORAMA_PUBLIC_FILES; | ||
| } | ||
| } | ||
| } |
65 changes: 65 additions & 0 deletions
65
panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicFileListener.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| package org.labkey.panoramapublic; | ||
|
|
||
| import org.jetbrains.annotations.NotNull; | ||
| import org.jetbrains.annotations.Nullable; | ||
| import org.labkey.api.data.Container; | ||
| import org.labkey.api.data.SQLFragment; | ||
| import org.labkey.api.exp.api.ExpData; | ||
| import org.labkey.api.exp.api.ExperimentService; | ||
| import org.labkey.api.files.FileListener; | ||
| import org.labkey.api.security.User; | ||
|
|
||
| import java.io.File; | ||
| import java.nio.file.Path; | ||
| import java.util.Collection; | ||
| import java.util.Collections; | ||
|
|
||
| public class PanoramaPublicFileListener implements FileListener | ||
| { | ||
|
|
||
| @Override | ||
| public String getSourceName() | ||
| { | ||
| return null; | ||
| } | ||
|
|
||
| @Override | ||
| public void fileCreated(@NotNull File created, @Nullable User user, @Nullable Container container) | ||
| { | ||
|
|
||
| } | ||
|
|
||
| @Override | ||
| public int fileMoved(@NotNull File src, @NotNull File dest, @Nullable User user, @Nullable Container container) | ||
| { | ||
| // Update any symlinks targeting the file | ||
| PanoramaPublicSymlinkManager.get().fireSymlinkUpdate(src.toPath(), dest.toPath(), container); | ||
|
|
||
| ExpData data = ExperimentService.get().getExpDataByURL(src, null); | ||
| if (null != data) | ||
| data.setDataFileURI(dest.toURI()); | ||
|
|
||
| return 0; | ||
| } | ||
|
|
||
| @Override | ||
| public void fileDeleted(@NotNull Path deleted, @Nullable User user, @Nullable Container container) | ||
| { | ||
| ExpData data = ExperimentService.get().getExpDataByURL(deleted, container); | ||
|
|
||
| if (null != data) | ||
| data.delete(user); | ||
| } | ||
|
|
||
| @Override | ||
| public Collection<File> listFiles(@Nullable Container container) | ||
| { | ||
| return Collections.emptyList(); | ||
| } | ||
|
|
||
| @Override | ||
| public SQLFragment listFilesQuery() | ||
| { | ||
| throw new UnsupportedOperationException("Not implemented"); | ||
| } | ||
| } |
Oops, something went wrong.