Merge pull request #401 from desci-labs/tay/automated-metadata

Automated Metadata
desci-labs · Jun 28, 2024 · 3dafe79 · 3dafe79
2 parents 0b62b74 + 884a151
commit 3dafe79
Show file tree

Hide file tree

Showing 27 changed files with 609 additions and 93 deletions.
diff --git a/.env.example b/.env.example
@@ -131,3 +131,7 @@ DOI_PREFIX=https://doi.org/10.62891
 CROSSREF_API=https://api.crossref.org
 CROSSREF_API_KEY=
 CROSSREF_EMAIL=
+
+# Automated metadata
+AUTOMATED_METADATA_API=http://host.docker.internal:5005
+AUTOMATED_METADATA_API_KEY=
diff --git a/.env.test b/.env.test
@@ -79,4 +79,8 @@ DOI_PREFIX=https://doi.org/10.62891
 # Cross ref api 
 CROSSREF_API=https://api.crossref.org
 CROSSREF_API_KEY=
-CROSSREF_EMAIL=
+CROSSREF_EMAIL=
+
+# Automated metadata
+AUTOMATED_METADATA_API=
+AUTOMATED_METADATA_API_KEY=
diff --git a/.nvmrc b/.nvmrc
@@ -1 +1 @@
-18.20.0
+20.8.1
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:18.20.0-bullseye
+FROM node:20.8.1-bullseye-slim
 
 VOLUME /root/.yarn
 

diff --git a/desci-models/src/automerge.ts b/desci-models/src/automerge.ts
@@ -39,6 +39,7 @@ export type ManifestActions =
   | { type: 'Upsert Components'; components: ResearchObjectV1Component[] }
   | { type: 'Delete Component'; path: string }
   | { type: 'Add Contributor'; author: ResearchObjectV1Author }
+  | { type: 'Add Contributors'; contributors: ResearchObjectV1Author[] }
   | { type: 'Remove Contributor'; contributorIndex: number }
   | { type: 'Add Contributors'; contributors: ResearchObjectV1Author[] }
   | { type: 'Pin Component'; path: string }

diff --git a/desci-repo/yarn.lock b/desci-repo/yarn.lock
@@ -3920,16 +3920,7 @@ string-argv@0.3.1:
   resolved "https://registry.npmjs.org/string-argv/-/string-argv-0.3.1.tgz"
   integrity sha512-a1uQGz7IyVy9YwhqjZIZu1c8JO8dNIe20xBmSS6qu9kv++k3JGzCVmprbNN5Kn+BgzD5E7YYwg1CcjuJMRNsvg==
 
-"string-width-cjs@npm:string-width@^4.2.0":
-  version "4.2.3"
-  resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz"
-  integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
-  dependencies:
-    emoji-regex "^8.0.0"
-    is-fullwidth-code-point "^3.0.0"
-    strip-ansi "^6.0.1"
-
-string-width@^4.1.0, string-width@^4.2.0:
+"string-width-cjs@npm:string-width@^4.2.0", string-width@^4.1.0, string-width@^4.2.0:
   version "4.2.3"
   resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz"
   integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
@@ -3999,14 +3990,7 @@ stringify-object@^3.3.0:
     is-obj "^1.0.1"
     is-regexp "^1.0.0"
 
-"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
-  version "6.0.1"
-  resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz"
-  integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
-  dependencies:
-    ansi-regex "^5.0.1"
-
-strip-ansi@^6.0.0, strip-ansi@^6.0.1:
+"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1:
   version "6.0.1"
   resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz"
   integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
@@ -4401,7 +4385,7 @@ workerpool@6.2.1:
   resolved "https://registry.npmjs.org/workerpool/-/workerpool-6.2.1.tgz"
   integrity sha512-ILEIE97kDZvF9Wb9f6h5aXK4swSlKGUcOEGiIYb2OOu/IrDU9iwj0fD//SsA6E5ibwJxpEvhullJY4Sl4GcpAw==
 
-"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0":
+"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0:
   version "7.0.0"
   resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz"
   integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
@@ -4419,15 +4403,6 @@ wrap-ansi@^6.2.0:
     string-width "^4.1.0"
     strip-ansi "^6.0.0"
 
-wrap-ansi@^7.0.0:
-  version "7.0.0"
-  resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz"
-  integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
-  dependencies:
-    ansi-styles "^4.0.0"
-    string-width "^4.1.0"
-    strip-ansi "^6.0.0"
-
 wrap-ansi@^8.1.0:
   version "8.1.0"
   resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz"

diff --git a/desci-server/kubernetes/deployment_dev.yaml b/desci-server/kubernetes/deployment_dev.yaml
@@ -84,6 +84,8 @@ spec:
           export CROSSREF_API={{ .Data.CROSSREF_API }}
           export CROSSREF_API_KEY={{ .Data.CROSSREF_API_KEY }}
           export CROSSREF_EMAIL={{ .Data.CROSSREF_EMAIL }}
+          export AUTOMATED_METADATA_API={{ .Data.AUTOMATED_METADATA_API }}
+          export AUTOMATED_METADATA_API_KEY={{ .Data.AUTOMATED_METADATA_API_KEY }}
           export DEBUG_TEST=0;
           echo "appfinish";
           {{- end -}}

diff --git a/desci-server/kubernetes/deployment_prod.yaml b/desci-server/kubernetes/deployment_prod.yaml
@@ -84,6 +84,8 @@ spec:
           export CROSSREF_API={{ .Data.CROSSREF_API }}
           export CROSSREF_API_KEY={{ .Data.CROSSREF_API_KEY }}
           export CROSSREF_EMAIL={{ .Data.CROSSREF_EMAIL }}
+          export AUTOMATED_METADATA_API={{ .Data.AUTOMATED_METADATA_API }}
+          export AUTOMATED_METADATA_API_KEY={{ .Data.AUTOMATED_METADATA_API_KEY }}
           export IGNORE_LINE=0;
           export DEBUG_TEST=0;
           echo "appfinish";

diff --git a/desci-server/kubernetes/deployment_staging.yaml b/desci-server/kubernetes/deployment_staging.yaml
@@ -96,6 +96,8 @@ spec:
           export CROSSREF_API={{ .Data.CROSSREF_API }}
           export CROSSREF_API_KEY={{ .Data.CROSSREF_API_KEY }}
           export CROSSREF_EMAIL={{ .Data.CROSSREF_EMAIL }}
+          export AUTOMATED_METADATA_API={{ .Data.AUTOMATED_METADATA_API }}
+          export AUTOMATED_METADATA_API_KEY={{ .Data.AUTOMATED_METADATA_API_KEY }}
           export DEBUG_TEST=0;
           echo "appfinish";
           {{- end -}}

diff --git a/desci-server/prisma/schema.prisma b/desci-server/prisma/schema.prisma
@@ -919,6 +919,7 @@ enum ActionType {
   UPDATE_ORCID_RECORD
   REMOVE_ORCID_WORK_RECORD
   ORCID_API_ERROR
+  // AUTOMATE_METADATA
 }
 
 enum ChainTransactionType {

diff --git a/desci-server/scripts/import-ipfs-content.sh b/desci-server/scripts/import-ipfs-content.sh
@@ -1,4 +1,16 @@
 # import required images from ipfs to local
 
+check_and_install() {
+  echo "check and install: $1"
+  # local shell_cmd = $1
+  if ! command -v $1 &> /dev/null;
+  then
+    apt-get install -y $1
+  fi
+}
+
+check_and_install "wget"
+check_and_install "curl"
+
 wget https://ipfs.desci.com/ipfs/bafkreih6yx7ywj7trvpp45vergrnytad7ezsku75tefyro4qrrcfrrmrt4 -O /tmp/cover.png
 curl -F file=@/tmp/cover.png "http://host.docker.internal:5001/api/v0/add?cid-version=1"
diff --git a/desci-server/src/controllers/data/update.ts b/desci-server/src/controllers/data/update.ts
@@ -7,6 +7,7 @@ import { RequestWithNode } from '../../middleware/authorisation.js';
 import { processExternalUrlDataToIpfs } from '../../services/data/externalUrlProcessing.js';
 import { processNewFolder, processS3DataToIpfs } from '../../services/data/processing.js';
 import { arrayXor, ensureUuidEndsWithDot } from '../../utils.js';
+
 export interface UpdateResponse {
   status?: number;
   rootDataCid?: string;
@@ -123,6 +124,7 @@ export const update = async (req: RequestWithNode, res: Response<UpdateResponse
         tree: tree,
         date: date,
       } = value as UpdateResponse;
+
       return res.status(200).json({
         rootDataCid: newRootCidString,
         manifest: updatedManifest,

diff --git a/desci-server/src/controllers/data/updateExternalCid.ts b/desci-server/src/controllers/data/updateExternalCid.ts
@@ -1,3 +1,4 @@
+import { ResearchObjectComponentSubtypes, ResearchObjectComponentType } from '@desci-labs/desci-models';
 import { User } from '@prisma/client';
 import { Response, Request } from 'express';
 
@@ -7,37 +8,30 @@ import { processExternalCidDataToIpfs } from '../../services/data/externalCidPro
 import { ensureUuidEndsWithDot } from '../../utils.js';
 
 import { ErrorResponse, UpdateResponse } from './update.js';
-import { ResearchObjectComponentSubtypes, ResearchObjectComponentType } from '@desci-labs/desci-models';
 
 export type ExternalCid = {
-  cid: string,
-  name: string,
+  cid: string;
+  name: string;
 };
 
 export type ExternalCidPayload = {
-  uuid: string,
-  contextPath: string,
-  externalCids: ExternalCid[],
-  componentType: ResearchObjectComponentType,
-  componentSubtype: ResearchObjectComponentSubtypes,
+  uuid: string;
+  contextPath: string;
+  externalCids: ExternalCid[];
+  componentType: ResearchObjectComponentType;
+  componentSubtype: ResearchObjectComponentSubtypes;
 };
 
 /**
  * Add an external UnixFS tree to the drive, without actually getting the files,
  * by fetching the leafless DAG and pinning it.
-*/
+ */
 export const updateExternalCid = async (
   req: Request<any, any, ExternalCidPayload>,
-  res: Response<UpdateResponse | ErrorResponse>
+  res: Response<UpdateResponse | ErrorResponse>,
 ) => {
   const owner = (req as any).user as User;
-  const {
-    uuid,
-    contextPath,
-    externalCids,
-    componentType,
-    componentSubtype
-  } = req.body;
+  const { uuid, contextPath, externalCids, componentType, componentSubtype } = req.body;
 
   const logger = parentLogger.child({
     // id: req.id,
@@ -73,7 +67,7 @@ export const updateExternalCid = async (
     externalCids,
     contextPath,
     componentType,
-    componentSubtype
+    componentSubtype,
   });
 
   if (ok) {
@@ -83,6 +77,7 @@ export const updateExternalCid = async (
       tree: tree,
       date: date,
     } = value as UpdateResponse;
+
     return res.status(200).json({
       manifest: updatedManifest,
       manifestCid: persistedManifestCid,

diff --git a/desci-server/src/controllers/data/utils.ts b/desci-server/src/controllers/data/utils.ts
@@ -1,4 +1,9 @@
-import { DRIVE_NODE_ROOT_PATH, DataComponent, ResearchObjectComponentType, ResearchObjectV1 } from '@desci-labs/desci-models';
+import {
+  DRIVE_NODE_ROOT_PATH,
+  DataComponent,
+  ResearchObjectComponentType,
+  ResearchObjectV1,
+} from '@desci-labs/desci-models';
 import { Node } from '@prisma/client';
 import axios from 'axios';
 import { v4 as uuid } from 'uuid';
@@ -30,7 +35,7 @@ function addDataToManifest({ manifest, dataFields, rootCid }: UpdatingManifestPa
       cid: rootCid,
       subMetadata: {},
       description: dataFields.description || undefined,
-      path: DRIVE_NODE_ROOT_PATH + `/${dataFields.title}`
+      path: DRIVE_NODE_ROOT_PATH + `/${dataFields.title}`,
     },
   };
   manifest.components.push(newDataComponent);
@@ -102,3 +107,14 @@ export function separateFileNameAndExtension(fileName: string): {
   const name = splitName.join('.');
   return { fileName: name, extension };
 }
+
+export const DOI_REGEX = /(https:\/\/doi.org\/)?(?<doi>10.\d{4,9}\/[-._;()/:A-Z0-9]+$)/i;
+export function isDoiLink(url: string): boolean {
+  try {
+    const matches = url.match(DOI_REGEX);
+    console.log('matches', matches);
+    return DOI_REGEX.test(url);
+  } catch (e) {
+    return false;
+  }
+}
diff --git a/desci-server/src/controllers/doi/schema.ts b/desci-server/src/controllers/doi/schema.ts
@@ -4,5 +4,6 @@ export const attachDoiSchema = z.object({
   body: z.object({
     uuid: z.string(),
     path: z.string().startsWith('root/', 'Invalid component path'),
+    publication: z.boolean().optional(),
   }),
 });