Skip to content

Commit

Permalink
various bug fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@jrobinso
jrobinso committed Oct 19, 2024
1 parent ed7b967 commit 43932a1
Show file tree
Hide file tree
Showing 8 changed files with 242 additions and 224 deletions.
2 changes: 1 addition & 1 deletion src/main/java/org/broad/igv/feature/genome/Genome.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ public Genome(GenomeConfig config) throws IOException {
} else if (config.getFastaURL() != null) {
String fastaPath = config.getFastaURL();
String indexPath = config.getIndexURL();
String gziIndexPath = config.getGziIndexURL() != null ? config.getGziIndexURL() : config.getCompressedIndexURL(); // Synonyms
String gziIndexPath = config.getGziIndexURL(); // Synonyms
uncachedSequence = fastaPath.endsWith(".gz") ?
new FastaBlockCompressedSequence(fastaPath, gziIndexPath, indexPath) :
new FastaIndexedSequence(fastaPath, indexPath);
Expand Down
151 changes: 61 additions & 90 deletions src/main/java/org/broad/igv/feature/genome/GenomeDownloadUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,17 +8,20 @@
import org.broad.igv.logging.LogManager;
import org.broad.igv.logging.Logger;
import org.broad.igv.ui.IGV;
import org.broad.igv.ui.util.MessageUtils;
import org.broad.igv.ui.commandbar.GenomeListManager;
import org.broad.igv.ui.util.download.Downloader;
import org.broad.igv.util.FileUtils;
import org.broad.igv.util.HttpUtils;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.lang.reflect.Field;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.List;
import java.util.Map;

/**
* Class of static functions for managing genome downloads
Expand All @@ -43,7 +46,9 @@ public static boolean isSequenceDownloadable(GenomeListItem item) {
return false;
}

public static File downloadGenome(GenomeConfig config, boolean downloadSequence, boolean downloadAnnotations) throws IOException {
public static File downloadGenome(GenomeConfig c, boolean downloadSequence, boolean downloadAnnotations) throws IOException {

GenomeConfig config = c.copy();

// Create a directory for the data files (sequence and annotations)
final File genomeDirectory = DirectoryManager.getGenomeCacheDirectory();
Expand All @@ -59,103 +64,74 @@ public static File downloadGenome(GenomeConfig config, boolean downloadSequence,
}

String relativeDataDirectory = config.getId() + "/";
File localFile;
if (downloadSequence) {

if (config.getTwoBitURL() != null) {

localFile = constructLocalFile(new URL(config.getTwoBitURL()), dataDirectory);

// .2bit files can be large, if it already exists confirm before replacing.
if(!localFile.exists() || MessageUtils.confirm("Replace existing file: " + localFile.getName() + "?")) {

download(new URL(config.getTwoBitURL()), localFile);

config.setTwoBitURL(relativeDataDirectory + localFile.getName());

// Null out urls not needed for .2bit seequences.
config.setFastaURL(null);
config.setIndexURL(null);
config.setGziIndexURL(null);

// The optional btree index. This is probably not needed for local file configurations.
if (config.getTwoBitBptURL() != null) {
localFile = download(new URL(config.getTwoBitBptURL()), dataDirectory);
config.setTwoBitBptURL(relativeDataDirectory + localFile.getName());
}
}
} else if (config.getFastaURL() != null) {

localFile = constructLocalFile(new URL(config.getFastaURL()), dataDirectory);
if (config.getTwoBitURL() != null) {

if(!localFile.exists() || MessageUtils.confirm("Replace existing file: " + localFile.getName() + "?")) {

localFile = download(new URL(config.getFastaURL()), localFile);

// Fasta files can be large, if it already exists confirm before replacing.
config.setFastaURL(relativeDataDirectory + localFile.getName());
if (config.getIndexURL() != null) {
localFile = download(new URL(config.getIndexURL()), dataDirectory);
config.setIndexURL(relativeDataDirectory + localFile.getName());
}
if (config.getGziIndexURL() != null) {
localFile = download(new URL(config.getGziIndexURL()), dataDirectory);
config.setGziIndexURL(relativeDataDirectory + localFile.getName());
}
}
URL url = new URL(config.getTwoBitURL());
File localFile;
if (downloadSequence) {
localFile = download(url, dataDirectory);
} else {
throw new RuntimeException("Sequence for genome " + config.getName() + " is not downloadable.");
localFile = constructLocalFile(url, dataDirectory); // It might be there from previous downloads
}
}
if (downloadAnnotations) {

if (config.getChromSizesURL() != null) {
localFile = download(new URL(config.getChromSizesURL()), dataDirectory);
config.setChromSizesURL(relativeDataDirectory + localFile.getName());
if (localFile.exists()) {
config.setTwoBitURL(relativeDataDirectory + localFile.getName());
// Null out urls not needed for .2bit sequences.
config.setFastaURL(null);
config.setIndexURL(null);
config.setGziIndexURL(null);
config.setTwoBitBptURL(null); // Not needed for local .2bit files
}
} else if (config.getFastaURL() != null) {

if (config.getCytobandBbURL() != null) {
localFile = download(new URL(config.getCytobandBbURL()), dataDirectory);
config.setCytobandBbURL(relativeDataDirectory + localFile.getName());
String[] fastaFields = {"fastaURL", "indexURL", "gziIndexURL", "compressedIndexURL"};
downloadAndUpdateConfig(downloadSequence, fastaFields, config, dataDirectory, relativeDataDirectory);

} else if (config.getCytobandURL() != null) {
localFile = download(new URL(config.getCytobandURL()), dataDirectory);
config.setCytobandURL(relativeDataDirectory + localFile.getName());
}
} else {
throw new RuntimeException("Sequence for genome " + config.getName() + " is not downloadable.");
}

if (config.getChromAliasBbURL() != null) {
localFile = download(new URL(config.getChromAliasBbURL()), dataDirectory);
config.setChromAliasBbURL(relativeDataDirectory + localFile.getName());
String[] annotationFields = {"chromAliasBbURL", "cytobandURL", "cytobandBbURL", "aliasURL", "chromSizesURL"};
downloadAndUpdateConfig(downloadAnnotations, annotationFields, config, dataDirectory, relativeDataDirectory);

} else if (config.getAliasURL() != null) {
localFile = download(new URL(config.getAliasURL()), dataDirectory);
config.setAliasURL(relativeDataDirectory + localFile.getName());
List<TrackConfig> trackConfigs = config.getTrackConfigs();
if (trackConfigs != null) {
String[] trackFields = {"url", "indexURL", "trixURL"};
for (TrackConfig trackConfig : trackConfigs) {
downloadAndUpdateConfig(downloadAnnotations, trackFields, trackConfig, dataDirectory, relativeDataDirectory);
}

List<TrackConfig> trackConfigs = config.getTrackConfigs();
}

if (trackConfigs != null) {
for (TrackConfig trackConfig : trackConfigs) {
if (trackConfig.getUrl() != null) {
localFile = download(new URL(trackConfig.getUrl()), dataDirectory);
trackConfig.setUrl(relativeDataDirectory + localFile.getName());
}
if (trackConfig.getIndexURL() != null) {
localFile = download(new URL(trackConfig.getIndexURL()), dataDirectory);
trackConfig.setIndexURL(relativeDataDirectory + localFile.getName());
File localGenomeFile = new File(genomeDirectory, config.getId() + ".json");
saveLocalGenome(config, localGenomeFile);
return localGenomeFile;

}

private static void downloadAndUpdateConfig(boolean downloadData, String[] fields, Object config, File dataDirectory, String relativeDataDirectory) {
URL url;
File localFile;
for (String f : fields) {
try {
Field field = config.getClass().getDeclaredField(f);
field.setAccessible(true);
Object urlField = field.get(config);
if (urlField != null) {
url = new URL(urlField.toString());
if (downloadData) {
localFile = download(url, dataDirectory);
} else {
localFile = constructLocalFile(url, dataDirectory); // It might be there from previous downloads
}
if(trackConfig.getTrixURL() != null) {
localFile = download(new URL(trackConfig.getTrixURL()), dataDirectory);
trackConfig.setTrixURL(relativeDataDirectory + localFile.getName());
if (localFile.exists()) {
field.set(config, relativeDataDirectory + localFile.getName());
}
}

} catch (Exception e) {
log.error(e);
}
}
File localGenomeFile = new File(genomeDirectory, config.getId() + ".json");
saveLocalGenome(config, localGenomeFile);
return localGenomeFile;

}

private static void saveLocalGenome(GenomeConfig genomeConfig, File localFile) throws IOException {
Expand All @@ -167,22 +143,17 @@ private static void saveLocalGenome(GenomeConfig genomeConfig, File localFile) t
}


private static File constructLocalFile(URL url, File directory) throws MalformedURLException {
public static File constructLocalFile(URL url, File directory) {
String path = url.getPath();
int idx = path.lastIndexOf('/');
String filename = idx < 0 ? path : path.substring(idx + 1);
return new File(directory, filename);
}

private static File download(URL url, File directory) throws MalformedURLException {
String path = url.getPath();
int idx = path.lastIndexOf('/');
String filename = idx < 0 ? path : path.substring(idx + 1);
File localFile = new File(directory, filename);
return getFile(url, localFile);
}

private static File getFile(URL url, File localFile) throws MalformedURLException {
File localFile = constructLocalFile(url, directory);

boolean success = Downloader.download(url, localFile, IGV.getInstance().getMainFrame());
if (!success) {
throw new RuntimeException("Download canceled");
Expand Down
15 changes: 7 additions & 8 deletions src/main/java/org/broad/igv/feature/genome/GenomeManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -288,26 +288,25 @@ public void restoreGenomeTracks(Genome genome) {
}


public GenomeListItem downloadGenome(GenomeListItem item, boolean downloadSequence, boolean downloadAnnotations) {
public File downloadGenome(GenomeListItem item, boolean downloadSequence, boolean downloadAnnotations) {

try {

if (item.getPath().endsWith(".genome")) {
File genomeFile = DotGenomeUtils.getDotGenomeFile(item.getPath()); // Will be downloaded if remote -- neccessary to unzip
item.setPath(genomeFile.getAbsolutePath());
return genomeFile;
} else {
JsonGenomeLoader loader = new JsonGenomeLoader(item.getPath());
GenomeConfig config = loader.loadGenomeConfig();
File downloadedGenome = GenomeDownloadUtils.downloadGenome(config, downloadSequence, downloadAnnotations);
item.setPath(downloadedGenome.getAbsolutePath());
return downloadedGenome;
}

} catch (Exception e) {
MessageUtils.showMessage("Error downloading genome: " + e.getMessage());
log.error("Error downloading genome " + item.getDisplayableName());
return null;
}

return item;
}


Expand All @@ -331,10 +330,10 @@ public void deleteDownloadedGenomes(List<GenomeListItem> removedValuesList) thro
File dataFileDirectory = new File(DirectoryManager.getGenomeCacheDirectory(), item.getId());
File localFasta = DotGenomeUtils.getLocalFasta(item.getId()); // (Legacy .genome convention)

if((dataFileDirectory.isDirectory() || localFasta != null) &&
MessageUtils.confirm("Delete downloaded data files?")){
if ((dataFileDirectory.isDirectory() || localFasta != null) &&
MessageUtils.confirm("Delete downloaded data files?")) {

if (dataFileDirectory.isDirectory()) {
if (dataFileDirectory.isDirectory()) {
try (Stream<Path> paths = Files.walk(dataFileDirectory.toPath())) {
paths.sorted(Comparator.reverseOrder())
.map(Path::toFile)
Expand Down
54 changes: 38 additions & 16 deletions src/main/java/org/broad/igv/feature/genome/load/GenomeConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,35 +3,42 @@
import com.google.gson.Gson;
import org.broad.igv.feature.Cytoband;
import org.broad.igv.feature.genome.Sequence;
import org.broad.igv.logging.LogManager;
import org.broad.igv.logging.Logger;

import java.util.LinkedHashMap;
import java.util.List;
import java.util.*;
import java.util.stream.Collectors;

/**
* A static json-like object representing a genome configuration. Emulates the javascript equivalent.
* This class was created to ease port of code from javascript.
*
* NOTE: The property names match names in the corresponding genome json files. They cannot be renamed.
* <p>
* Objects of this class are created by two paths:
* (1) GSON deserializtion of a genome json file, and
* (2) parsing track hub files.
* (1) GSON deserializtion of a genome json file, and
* (2) parsing track hub files.
*/

public class GenomeConfig {
public class GenomeConfig implements Cloneable {

private static Logger log = LogManager.getLogger(GenomeConfig.class);

private String id;
private String name;
private String fastaURL;
private String indexURL;
private String gziIndexURL;
private String compressedIndexURL;
private String compressedIndexURL; // Used by reflection from GSON. Do not remove
private String twoBitURL;
private String twoBitBptURL;
private String nameSet;
private boolean wholeGenomeView = true;
private String defaultPos;
private String description;
private String blat;
private String chromAliasBbURL;
private String twoBitBptURL;

private String infoURL;
private String cytobandURL;
private String cytobandBbURL;
Expand Down Expand Up @@ -95,21 +102,13 @@ public void setIndexURL(String indexURL) {
}

public String getGziIndexURL() {
return gziIndexURL;
return this.gziIndexURL != null ? gziIndexURL : compressedIndexURL;
}

public void setGziIndexURL(String gziIndexURL) {
this.gziIndexURL = gziIndexURL;
}

public String getCompressedIndexURL() {
return compressedIndexURL;
}

public void setCompressedIndexURL(String compressedIndexURL) {
this.compressedIndexURL = compressedIndexURL;
}

public String getTwoBitURL() {
return twoBitURL;
}
Expand Down Expand Up @@ -285,4 +284,27 @@ public List<List<String>> getChromAliases() {
public void setChromAliases(List<List<String>> chromAliases) {
this.chromAliases = chromAliases;
}

public GenomeConfig copy() {
return this.clone();
}

protected GenomeConfig clone() {
try {
GenomeConfig clone = (GenomeConfig) super.clone();

if (this.tracks != null) {
clone.tracks = new ArrayList<>();
for (TrackConfig trackConfig : this.tracks) {
clone.tracks.add(trackConfig.clone());
}
}

return clone;
} catch (CloneNotSupportedException e) {
log.error("Cloning not supported", e);
return this;
}
}

}
3 changes: 0 additions & 3 deletions src/main/java/org/broad/igv/feature/genome/load/JsonGenomeLoader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,9 +130,6 @@ private GenomeConfig fixPaths(GenomeConfig config) {
if (config.getGziIndexURL() != null) {
config.setGziIndexURL(FileUtils.getAbsolutePath(config.getGziIndexURL(), genomePath));
}
if (config.getCompressedIndexURL() != null) {
config.setCompressedIndexURL(FileUtils.getAbsolutePath(config.getCompressedIndexURL(), genomePath));
}
if (config.getCytobandURL() != null) {
config.setCytobandURL(FileUtils.getAbsolutePath(config.getCytobandURL(), genomePath));
}
Expand Down
Loading

0 comments on commit 43932a1

Please sign in to comment.