From c0230888c3e3d5b4ac8f19397d1c74e462621234 Mon Sep 17 00:00:00 2001 From: laurenwalker Date: Fri, 12 Jul 2019 15:36:06 -0400 Subject: [PATCH] Fixe major issue where the system metadata of each object in the package was retrieved when the MetadataView was loaded. Closes #978 --- src/js/collections/DataPackage.js | 204 ++++++++++++++++++++++++------ src/js/collections/SolrResults.js | 29 +++-- src/js/models/DataONEObject.js | 16 ++- src/js/models/SolrResult.js | 2 +- src/js/views/EditorView.js | 31 ++--- src/js/views/MetadataView.js | 56 +++++++- 6 files changed, 257 insertions(+), 81 deletions(-) diff --git a/src/js/collections/DataPackage.js b/src/js/collections/DataPackage.js index da6031d71..3f26b123d 100644 --- a/src/js/collections/DataPackage.js +++ b/src/js/collections/DataPackage.js @@ -2,8 +2,11 @@ "use strict"; define(['jquery', 'underscore', 'backbone', 'rdflib', "uuid", "md5", + 'collections/SolrResults', + 'models/filters/Filter', 'models/DataONEObject', 'models/metadata/ScienceMetadata', 'models/metadata/eml211/EML211'], - function($, _, Backbone, rdf, uuid, md5, DataONEObject, ScienceMetadata, EML211) { + function($, _, Backbone, rdf, uuid, md5, SolrResults, Filter, + DataONEObject, ScienceMetadata, EML211) { /* A DataPackage represents a hierarchical collection of @@ -57,6 +60,19 @@ define(['jquery', 'underscore', 'backbone', 'rdflib', "uuid", "md5", // The nesting level in a data package hierarchy nodeLevel: 0, + /** + * @type {SolrResults} - The SolrResults collection associated with this DataPackage. + * This can be used to fetch the package from Solr by passing the 'fromIndex' option + * to fetch(). + */ + solrResults: new SolrResults(), + + /** + * @type {Filter} - A Filter model that should filter the Solr index for only the + * objects aggregated by this package. + */ + filterModel: null, + //Define the namespaces used in the RDF XML namespaces: { RDF: "http://www.w3.org/1999/02/22-rdf-syntax-ns#", @@ -111,6 +127,17 @@ define(['jquery', 'underscore', 'backbone', 'rdflib', "uuid", "md5", } this.id = this.packageModel.id; + //Create a Filter for this DataPackage using the id + this.filterModel = new Filter({ + fields: ["resourceMap"], + values: [this.id], + matchSubstring: false + }); + //If the id is ever changed, update the id in the Filter + this.listenTo(this.packageModel, "change:id", function(){ + this.filterModel.set("values", [this.packageModel.get("id")]); + }); + this.on("add", this.handleAdd); this.on("add", this.triggerComplete); this.on("successSaving", this.updateRelationships); @@ -320,14 +347,41 @@ define(['jquery', 'underscore', 'backbone', 'rdflib', "uuid", "md5", } }, - /* + /** * Overload fetch calls for a DataPackage + * + * @param {Object} [options] - Optional options for this fetch that get sent with the XHR request + * @property {boolean} fetchModels - If false, this fetch will not fetch + * each model in the collection. It will only get the resource map object. + * @property {boolean} fromIndex - If true, the collection will be fetched from Solr rather than + * fetching the system metadata of each model. Useful when you only need to retrieve limited information about + * each package member. Set query-specific parameters on the `solrResults` SolrResults set on this collection. */ fetch: function(options) { //Fetch the system metadata for this resource map this.packageModel.fetch(); + if(typeof options == "object"){ + + //If the fetchModels property is set to false, + if(options.fetchModels === false){ + //Save the property to the Collection itself so it is accessible in other functions + this.fetchModels = false; + //Remove the property from the options Object since we don't want to send it with the XHR + delete options.fetchModels; + + this.once("reset", this.triggerComplete); + } + //If the fetchFromIndex property is set to true + else if( options.fromIndex ){ + + this.fetchFromIndex(); + return; + + } + } + //Set some custom fetch options var fetchOptions = _.extend({dataType: "text"}, options); @@ -396,16 +450,20 @@ define(['jquery', 'underscore', 'backbone', 'rdflib', "uuid", "md5", //Create a DataONEObject model to represent this collection member and add to the collection if(!_.contains(this.pluck("id"), memberPID)){ - memberModel = this.add(new DataONEObject({ + memberModel = new DataONEObject({ id: memberPID, resourceMap: [this.packageModel.get("id")], collections: [this] - }), { silent: true }); + }); + + models.push(memberModel); } //If the model already exists, add this resource map ID to it's list of resource maps else{ memberModel = this.get(memberPID); + models.push(memberModel); + var rMaps = memberModel.get("resourceMap"); if(rMaps && Array.isArray(rMaps) && !_.contains(rMaps, this.packageModel.get("id"))) rMaps.push(this.packageModel.get("id")); else if(rMaps && !Array.isArray(rMaps)) rMaps = [rMaps, this.packageModel.get("id")]; @@ -444,7 +502,8 @@ define(['jquery', 'underscore', 'backbone', 'rdflib', "uuid", "md5", this.originalIsDocBy[scidataID] = _.uniq([this.originalIsDocBy[scidataID], scimetaID]); //Find the model in this collection for this data object - var dataObj = this.get(scidataID); + //var dataObj = this.get(scidataID); + var dataObj = _.find(models, function(m){ return m.get("id") == scidataID }); if(dataObj){ //Get the isDocumentedBy field @@ -465,54 +524,62 @@ define(['jquery', 'underscore', 'backbone', 'rdflib', "uuid", "md5", memberPIDs = _.difference(memberPIDs, sciMetaPids); _.each(_.uniq(sciMetaPids), function(id){ memberPIDs.unshift(id); }); - //Retrieve the model for each member - _.each(memberPIDs, function(pid){ + //Don't fetch each member model if the fetchModels property on this Collection is set to false + if( this.fetchModels !== false ){ - memberModel = this.get(pid); + //Add the models to the collection now, silently + //this.add(models, {silent: true}); - var collection = this; + //Retrieve the model for each member + _.each(models, function(memberModel){ - memberModel.fetch(); - memberModel.once("sync", - function(oldModel){ + var collection = this; - //Get the right model type based on the model values - var newModel = collection.getMember(oldModel); + memberModel.fetch(); + memberModel.once("sync", + function(oldModel){ - //If the model type has changed, then mark the model as unsynced, since there may be custom fetch() options for the new model - if(oldModel.type != newModel.type){ + //Get the right model type based on the model values + var newModel = collection.getMember(oldModel); - // DataPackages shouldn't be fetched until we support nested packages better in the UI - if(newModel.type == "DataPackage"){ - //Trigger a replace event so other parts of the app know when a model has been replaced with a different type - oldModel.trigger("replace", newModel); - } - else{ - newModel.set("synced", false); + //If the model type has changed, then mark the model as unsynced, since there may be custom fetch() options for the new model + if(oldModel.type != newModel.type){ + + // DataPackages shouldn't be fetched until we support nested packages better in the UI + if(newModel.type == "DataPackage"){ + //Trigger a replace event so other parts of the app know when a model has been replaced with a different type + oldModel.trigger("replace", newModel); + } + else{ + newModel.set("synced", false); - newModel.fetch(); - newModel.once("sync", function(fetchedModel){ - fetchedModel.set("synced", true); - collection.add(fetchedModel, { merge: true }); + newModel.fetch(); + newModel.once("sync", function(fetchedModel){ + fetchedModel.set("synced", true); - //Trigger a replace event so other parts of the app know when a model has been replaced with a different type - oldModel.trigger("replace", newModel); + //Remove the model from the collection and add it back + collection.remove(oldModel); + collection.add(fetchedModel); - if(newModel.type == "EML") - collection.trigger("add:EML"); - }); + //Trigger a replace event so other parts of the app know when a model has been replaced with a different type + oldModel.trigger("replace", newModel); + + if(newModel.type == "EML") + collection.trigger("add:EML"); + }); + } } - } - else{ - newModel.set("synced", true); - collection.add(newModel, { replace: true }); + else{ + newModel.set("synced", true); + collection.add(newModel, { merge: true }); - if(newModel.type == "EML") - collection.trigger("add:EML"); - } - }); + if(newModel.type == "EML") + collection.trigger("add:EML"); + } + }); - }, this); + }, this); + } } catch (error) { console.log(error); @@ -1486,6 +1553,17 @@ define(['jquery', 'underscore', 'backbone', 'rdflib', "uuid", "md5", }, triggerComplete: function(model){ + + //If the last fetch did not fetch the models of the collection, then mark as complete now. + if(this.fetchModels === false){ + // Delete the fetchModels property since it is set only once per fetch. + delete this.fetchModels; + + this.trigger("complete", this); + + return; + } + //Check if the collection is done being retrieved var notSynced = this.reject(function(m){ return (m.get("synced") || m.get("id") == model.get("id")); @@ -2630,7 +2708,49 @@ define(['jquery', 'underscore', 'backbone', 'rdflib', "uuid", "md5", }, - /* + /** + * Fetches this DataPackage from the Solr index by using a SolrResults collection + * and merging the models in. + */ + fetchFromIndex: function(){ + + if( typeof this.solrResults == "undefined" || !this.solrResults ){ + this.solrResults = new SolrResults(); + } + + //If no query is set yet, use the FilterModel associated with this DataPackage + if( !this.solrResults.currentquery.length ){ + this.solrResults.currentquery = this.filterModel.getQuery(); + } + + this.listenToOnce(this.solrResults, "reset", function(solrResults){ + solrResults.each(function(solrResult){ + + var modelInDataPackage = this.findWhere({ id: solrResult.get("id") }); + + if( modelInDataPackage ){ + + var fieldsToExtract = this.solrResults.fields ? this.solrResults.fields.split(",") : []; + + if( fieldsToExtract.length ){ + var valuesFromSolr = _.pick(solrResult.toJSON(), fieldsToExtract); + + modelInDataPackage.set(valuesFromSolr); + } + + } + + }, this); + + this.trigger("complete"); + }); + + //Query the index for this data package + this.solrResults.query(); + + }, + + /** * Update the relationships in this resource map when its been udpated */ updateRelationships: function(){ diff --git a/src/js/collections/SolrResults.js b/src/js/collections/SolrResults.js index 8b01a08d2..ebb41801c 100644 --- a/src/js/collections/SolrResults.js +++ b/src/js/collections/SolrResults.js @@ -70,15 +70,20 @@ define(['jquery', 'underscore', 'backbone', 'models/SolrHeader', 'models/SolrRes } //create the query url - var endpoint = MetacatUI.appModel.get('queryServiceUrl') + - "fl=" + this.fields + - "&q=" + this.currentquery + - "&sort=" + this.sort + - "&rows=" + this.rows + - "&start=" + this.start + - "&facet=true&facet.sort=index" + facetFields + - stats + - "&wt=json"; + var endpoint = MetacatUI.appModel.get('queryServiceUrl') + "q=" + this.currentquery; + + if(this.fields) + endpoint += "&fl=" + this.fields; + if(this.sort) + endpoint += "&sort=" + this.sort; + if( typeof this.rows == "number" || (typeof this.rows == "string" && this.rows.length)) + endpoint += "&rows=" + this.rows; + if( typeof this.start == "number" || (typeof this.start == "string" && this.start.length)) + endpoint += "&start=" + this.start; + if( this.facet.length > 0 ) + endpoint += "&facet=true&facet.sort=index" + facetFields; + + endpoint += stats + "&wt=json"; return endpoint; }, @@ -151,14 +156,14 @@ define(['jquery', 'underscore', 'backbone', 'models/SolrHeader', 'models/SolrRes }, query: function(newquery) { - if (this.currentquery != newquery) { + + if(typeof newquery != "undefined" && this.currentquery != newquery){ this.currentquery = newquery; this.start = 0; - } + var fetchOptions = this.createFetchOptions(); this.fetch(fetchOptions); - //this.fetch({data: {start: this.start}, reset: true}); }, setQuery: function(newquery) { diff --git a/src/js/models/DataONEObject.js b/src/js/models/DataONEObject.js index e693ec7da..f706870d8 100644 --- a/src/js/models/DataONEObject.js +++ b/src/js/models/DataONEObject.js @@ -51,8 +51,8 @@ define(['jquery', 'underscore', 'backbone', 'uuid', 'he', 'collections/AccessPol dateModified: null, id: "urn:uuid:" + uuid.v4(), sizeStr: null, - type: null, // Data, Metadata, or DataPackage - formatType: null, + type: "", // Data, Metadata, or DataPackage + formatType: "", metadataEntity: null, // A model that represents the metadata for this file, e.g. an EMLEntity model latestVersion: null, isDocumentedBy: null, @@ -236,7 +236,7 @@ define(['jquery', 'underscore', 'backbone', 'uuid', 'he', 'collections/AccessPol } //Merge with the options passed to this function - var fetchOptions = _.extend(solrOptions, options); + var fetchOptions = _.extend(options, solrOptions); } else if(typeof options != "undefined"){ //Use custom options for retreiving XML @@ -1028,7 +1028,7 @@ define(['jquery', 'underscore', 'backbone', 'uuid', 'he', 'collections/AccessPol handleChange: function(model, options) { if(!model) var model = this; - var sysMetaAttrs = ["serialVersion", "identifier", "formatId", "size", "checksum", + var sysMetaAttrs = ["serialVersion", "identifier", "formatId", "formatType", "size", "checksum", "checksumAlgorithm", "submitter", "rightsHolder", "accessPolicy", "replicationAllowed", "replicationPolicy", "obsoletes", "obsoletedBy", "archived", "dateUploaded", "dateSysMetadataModified", "originMemberNode", "authoritativeMemberNode", "replica", "seriesId", "mediaType", "fileName"], @@ -1072,6 +1072,10 @@ define(['jquery', 'underscore', 'backbone', 'uuid', 'he', 'collections/AccessPol */ updateUploadStatus: function(){ + if( !this.get("synced") ){ + return; + } + //Add this item to the queue if((this.get("uploadStatus") == "c") || (this.get("uploadStatus") == "e") || !this.get("uploadStatus")){ this.set("uploadStatus", "q"); @@ -1388,7 +1392,7 @@ define(['jquery', 'underscore', 'backbone', 'uuid', 'he', 'collections/AccessPol //Determine the type via provONE var instanceOfClass = this.get("prov_instanceOfClass"); - if(typeof instanceOfClass !== "undefined" && instanceOfClass.length){ + if(typeof instanceOfClass !== "undefined" && Array.isArray(instanceOfClass) && instanceOfClass.length){ var programClass = _.filter(instanceOfClass, function(className){ return (className.indexOf("#Program") > -1); }); @@ -1481,7 +1485,7 @@ define(['jquery', 'underscore', 'backbone', 'uuid', 'he', 'collections/AccessPol }, isSoftware: function(){ - //The list of formatIds that are programs + //The list of formatIds that are programs var softwareIds = ["text/x-python", "text/x-rsrc", "text/x-matlab", diff --git a/src/js/models/SolrResult.js b/src/js/models/SolrResult.js index 3e42cc3b5..77a9a40a0 100644 --- a/src/js/models/SolrResult.js +++ b/src/js/models/SolrResult.js @@ -35,7 +35,7 @@ define(['jquery', 'underscore', 'backbone'], datasource: null, rightsHolder: null, size: 0, - type: null, + type: "", url: null, obsoletedBy: null, geohash_9: null, diff --git a/src/js/views/EditorView.js b/src/js/views/EditorView.js index f39756057..b185184f1 100644 --- a/src/js/views/EditorView.js +++ b/src/js/views/EditorView.js @@ -65,19 +65,23 @@ define(['underscore', else var model = new ScienceMetadata({ id: this.pid }); - // Once the ScienceMetadata is populated, populate the associated package - this.model = model; - - //Listen for the replace event on this model - var view = this; - this.listenTo(this.model, "replace", function(newModel){ - if(view.model.get("id") == newModel.get("id")){ - view.model = newModel; - view.setListeners(); - } - }); + // Once the ScienceMetadata is populated, populate the associated package + this.model = model; + + //Listen for the replace event on this model + var view = this; + this.listenTo(this.model, "replace", function(newModel){ + if(this.model.get("id") == newModel.get("id")){ + + this.model = newModel; + this.setListeners(); - this.setListeners(); + //If this model has been upgraded to an EML doc, render the metadata + this.renderMember(newModel); + } + }); + + this.setListeners(); }, /* Render the view */ @@ -365,13 +369,10 @@ define(['underscore', break; case "Metadata": - - // this.renderDataPackageItem(model, collection, options); this.renderMetadata(model, collection, options); break; case "Data": - //this.renderDataPackageItem(model, collection, options); break; default: diff --git a/src/js/views/MetadataView.js b/src/js/views/MetadataView.js index 1a1d49377..441fde68f 100644 --- a/src/js/views/MetadataView.js +++ b/src/js/views/MetadataView.js @@ -141,14 +141,32 @@ define(['jquery', //var metadataModel = new ScienceMetadata({ id: this.pid }); // Once the ScienceMetadata is populated, populate the associated package //this.metadataModel = metadataModel; + + //Create a DataONEObject model to use in the DataPackage collection. + // var combinedAttr = _.extend(this.model.toJSON()); + var solrResultDefaults = this.model.defaults, + solrResultAttr = this.model.toJSON(), + omitKeys = []; + + _.each(solrResultAttr, function(val, key){ + if( solrResultDefaults[key] === val ) + omitKeys.push(key); + }); + + var dataOneObject = new ScienceMetadata(_.omit(solrResultAttr, omitKeys)); + // Create a new data package with this id - this.dataPackage = new DataPackage([], {id: pid}); + this.dataPackage = new DataPackage([dataOneObject], {id: pid}); + //Fetch the data package. DataPackage.parse() triggers 'complete' - this.dataPackage.fetch(); - this.listenTo(this.dataPackage, "complete", function() { + this.dataPackage.fetch({ + fetchModels: false + }); + + this.listenToOnce(this.dataPackage, "complete", function() { // parseProv triggers "queryComplete" this.dataPackage.parseProv(); - this.checkForProv(); + this.checkForProv(); }); }, /* @@ -1202,6 +1220,32 @@ define(['jquery', editModeOn = false; } + if( editModeOn ){ + //If none of the models in this package have the formatId attributes, + // we should fetch the DataPackage since it likely has only had a shallow fetch so far + var formats = _.compact(dataPackage.pluck("formatId")); + + if( formats.length < dataPackage.length ){ + + this.listenToOnce(dataPackage, "complete", function(){ + this.drawProvCharts(dataPackage); + }); + + + dataPackage.solrResults.currentquery = dataPackage.filterModel.getQuery() + + "%20AND%20-formatType:METADATA"; + dataPackage.solrResults.fields = "id,seriesId,formatId,fileName"; + dataPackage.solrResults.rows = dataPackage.length; + dataPackage.solrResults.sort = null; + dataPackage.solrResults.start = 0; + dataPackage.solrResults.facet = []; + dataPackage.solrResults.stats = null; + dataPackage.fetch({ fromIndex: true }); + + return; + } + } + var view = this; //Draw two flow charts to represent the sources and derivations at a package level var packageSources = dataPackage.sourcePackages; @@ -1234,7 +1278,9 @@ define(['jquery', //Draw the provenance charts for each member of this package at an object level _.each(dataPackage.toArray(), function(member, i){ // Don't draw prov charts for metadata objects. - if(member.get("type").toLowerCase() == "metadata") return; + if(member.get("type").toLowerCase() == "metadata" || member.get("formatType").toLowerCase() == "metadata"){ + return; + } var entityDetailsSection = view.findEntityDetailsContainer(member); if( !entityDetailsSection ){