From 482f6270f569af5fbfac7adb3ebaa5dfb07c8b15 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Wed, 10 Jun 2020 12:53:52 -0500 Subject: [PATCH] Add provenance to job feedback (plots, images, etc). Closes #1728 (#1732) * Update pipeline seed with metadata provenance * Fix corrupt releases file * Record provenance for metadata. Closes #1728 * Add loadChildren helper to Metadata --- src/plugins/ExecuteJob/ExecuteJob.Metadata.js | 13 +++++++++---- src/plugins/ExecuteJob/metadata/Figure.js | 3 ++- src/plugins/ExecuteJob/metadata/Metadata.js | 8 ++++++++ src/seeds/pipeline/pipeline.webgmex | Bin 33958 -> 35336 bytes src/seeds/pipeline/releases.jsonl | 3 ++- 5 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/plugins/ExecuteJob/ExecuteJob.Metadata.js b/src/plugins/ExecuteJob/ExecuteJob.Metadata.js index 8fc6cb080..fdb216ca9 100644 --- a/src/plugins/ExecuteJob/ExecuteJob.Metadata.js +++ b/src/plugins/ExecuteJob/ExecuteJob.Metadata.js @@ -16,7 +16,7 @@ define([ const nodeId = this.core.getPath(job); this.lastAppliedCmd[nodeId] = 0; const metadata = await this.getMetadataNodes(job); - await Promise.all(metadata.map(node => this.resetMetadataNode(node))); + await Promise.all(metadata.map(node => this.resetMetadataNode(node, job))); }; ExecuteJob.prototype.clearOldMetadata = async function (job) { @@ -39,13 +39,16 @@ define([ } }; - ExecuteJob.prototype.resetMetadataNode = async function (node) { + ExecuteJob.prototype.resetMetadataNode = async function (node, job) { const children = await this.core.loadChildren(node); children.forEach(child => this.core.deleteNode(child)); const attributes = this.core.getAttributeNames(node) .filter(attr => attr !== 'id'); attributes.forEach(attr => this.core.delAttribute(node, attr)); + + const op = await this.getOperation(job); + await this.recordProvenance(node, op); }; ExecuteJob.prototype.getMetadataNodes = async function (job) { @@ -100,13 +103,14 @@ define([ const MetadataClass = Metadata.getClassForCommand(cmd); const metadata = await this.getMetadataNodes(job); const node = metadata.find(node => this.core.getAttribute(node, 'id')) || - this.createNodeForMetadata(MetadataClass, job, id); + await this.createNodeForMetadata(MetadataClass, job, id); const md = new MetadataClass(node, this.core, this.META); await md.update(content); }; - ExecuteJob.prototype.createNodeForMetadata = function (MetadataClass, job, id) { + ExecuteJob.prototype.createNodeForMetadata = async function (MetadataClass, job, id) { + const op = await this.getOperation(job); const base = this.META[MetadataClass.getMetaType()]; const msg = `Metadata type not found for ${MetadataClass.name}: ` + `${MetadataClass.getMetaType()}`; @@ -115,6 +119,7 @@ define([ const node = this.core.createNode({base, parent: job}); this.core.setAttribute(node, 'id', id); + await this.recordProvenance(node, op); return node; }; diff --git a/src/plugins/ExecuteJob/metadata/Figure.js b/src/plugins/ExecuteJob/metadata/Figure.js index 7d1cfca3c..891a3c5b6 100644 --- a/src/plugins/ExecuteJob/metadata/Figure.js +++ b/src/plugins/ExecuteJob/metadata/Figure.js @@ -52,7 +52,8 @@ define([ } async clearSubGraphs() { - const subGraphs = await this.core.loadChildren(this.node); + const subGraphs = await this.loadChildren(); + subGraphs.forEach(subGraph => this.core.deleteNode(subGraph)); } diff --git a/src/plugins/ExecuteJob/metadata/Metadata.js b/src/plugins/ExecuteJob/metadata/Metadata.js index e1420ba3d..ae6cc4b88 100644 --- a/src/plugins/ExecuteJob/metadata/Metadata.js +++ b/src/plugins/ExecuteJob/metadata/Metadata.js @@ -13,6 +13,14 @@ define([ throw new Error('not implemented!'); } + async loadChildren() { + const provPath = this.core.getPointerPath(this.node, 'provenance'); + const children = (await this.core.loadChildren(this.node)) + .filter(node => this.core.getPath(node) !== provPath); + + return children; + } + static getCommand() { throw new Error('not implemented!'); } diff --git a/src/seeds/pipeline/pipeline.webgmex b/src/seeds/pipeline/pipeline.webgmex index 3c28f18bd77100901a9f9754cfb464f386f17305..46c7525c315f6fbe22d5a9a45c615311e0227934 100644 GIT binary patch delta 1474 zcma)+&ube;6vwraW4rm~w6$?%A52C`EITVSJ3Bvi)tWRvWH$*lYe?!QCOES@Gqqdn zM3Jh<1f>q-;6OrtO#g$D(jF2Bd&$9&9(u{KkV_6J^w2}0P)g~!9ce?0T?%_x81{YV zecycM&HMURcH_(Jp5Sz@XE4*B$z;Cz_WAuihl?0GSV%-95|oONLr4Ty%5`DrP>w>E zxe5}j?q3W(^SKi-%p**QFO6J6kGhBo%oLW~#aJQ_NkV}QmUw+h)Y-cIaWR;*rld291= zLDUJQM2^EFg%MRu2}jBZ*bih$-^{%lSAx0NWbk~17-N`8k4BM?nNZTDz5_j1MGlHL zdVFirA&R-kS5yjY4jw{^O&KEcq4H3~jcF7L3Y97T_w`cz>yDC@nr?iw@5G?~ZD?3O z8aX)3Yqh27M;&-8awU{kKn0Y~l>=RSGe3N$;d#fR|HhsSpXlWa>lx2=r^l z_r(vVo>;}S-4I_t^34t_(Bswp%ja5Wt247(;ju0{RbH&iM!meavfNr;t(V!1tR1gz zRR&$<+&1NzvwY^nwhIg53)M+0b^AS@=2qDTf$4)n+AfUuez5jVY6x9~-(UQBmkqk- zTm%)QZ#3SHH+j?2pS0f6)xM&B)*8`wTJyaL1igN0qVF{i{R129O_*r7Het2nKgET8 z#gy+W?AqV`d~u8Y*cbctK=ZI=>)Wl5|Iag_+wqK9b8}}gEuM}e3zeXr!k%vi2N&2( z8fgZE5Rq`H;KU3Hu*z4Y!9Ks-SPHIOYRXl<8tKbZqq`l-x@}L9c_)9J9MMB_v-(Go z*Y{54_4(*=Z_-PqviJ3@Ao}V;hu=XXIS~J`KA$`EqVK2H^;?-t=L=3-*;4MGxg7i( O)0c0;-ls;A$@~qYP`Nt* delta 922 zcma)*O-PhM7{}jtcXew+`>>)94>m@vy5jpe^UhjmD`Y6aLihoZIiL7}L~W#DTbD%? zPx90;s1D+xh?+VGQXQgO7q1c2p+kpG6?p8mqOLE)!0^n&{Qm#{%q-qbtuCb+%7f|5 zertziS>KmmtTc4wH782yNH|0i5h;aTL9ybDX+b?nT%{QELM~Ru%hQp^n4=u-;6OXt z!Ct`K$kS2>F7ap(m$Bx|r>KA?SY<+0P;nkz;ZsUnkCzuwYf>00z}{?UTdNykPsuP+ zTu{jaiFqgjf~nF}P&e@1Kr$jQHCOEoyB%XkQs(;s;h1CJ;Y`F{Jfg5B*pEj@u^Wc5 zJecqAgj)LJ$$)A?m7+{=O0eq$z7~?i#$^n_zK99&0n{vJsBOO0joOLxf)!POPg9Ms zcxsrqVvPI~Vx{bQ2X_&w&rL+pfaX(;YA_TVS*J zJfAp$$K8k1iM!$R!U0J2oX>@S*i=kjhjVSvJADM+4SBH28{wWR!R^{{xHP=Wywv4A`GoI*$E1VMH#-K8 wcoV#yrRM$Idd>_jjHFN8sg>?G*DPy$mIv)rXZqiNEC1hv%_XOPIBv15Uk7~~K>z>% diff --git a/src/seeds/pipeline/releases.jsonl b/src/seeds/pipeline/releases.jsonl index b1aab2ab2..5011f29b8 100644 --- a/src/seeds/pipeline/releases.jsonl +++ b/src/seeds/pipeline/releases.jsonl @@ -1 +1,2 @@ -{"version":"0.20.0","changelog":"Add provenance info to Data nodes"} \ No newline at end of file +{"version":"0.20.0","changelog":"Add provenance info to Data nodes"} +{"version":"0.21.0","changelog":"Add provenance to metadata (via WithProvenance mixin)"}