Adjust to PubChemRDF from uniprot-kg

dvc.yaml

@@ -14,12 +14,12 @@ stages:
     deps:
       - stages/00_invalidate.sh
     outs:
-      - checksum
+      - void
   download:
     cmd: stages/01_download.sh
     deps:
       - stages/01_download.sh
-      - checksum
+      - void
     outs:
       - download:
           persist: true

stages/00_invalidate.sh

@@ -6,16 +6,16 @@
 localpath=$(pwd)
 echo "Local path: $localpath"
 
-# Define the release URL for the dataset
-checksum_url="https://ftp.uniprot.org/pub/databases/uniprot/current_release/rdf/RELEASE.metalink"
+# Define the VoID URL for the dataset
+void_url="https://ftp.ncbi.nlm.nih.gov/pubchem/RDF/void.ttl"
 
-# Create the checksum directory
-checksumpath="$localpath/checksum"
-echo "Checksum path: $checksumpath"
-mkdir -p "$checksumpath"
-cd $checksumpath;
+# Create the VoID directory
+voidpath="$localpath/void"
+echo "VoID path: $voidpath"
+mkdir -p "$voidpath"
+cd $voidpath;
 
 # Download file
-wget -P $checksumpath $checksum_url
+wget -P $voidpath $void_url
 
 echo "Download done."

stages/01_download.sh

@@ -6,8 +6,10 @@
 localpath=$(pwd)
 echo "Local path: $localpath"
 
-# Define the release URL for the dataset
-metalink_url="https://ftp.uniprot.org/pub/databases/uniprot/current_release/rdf/RELEASE.metalink"
+# Define the FTP URL for the dataset
+# https://ftp.ncbi.nlm.nih.gov/pubchem/RDF/
+# ftp://ftp.ncbi.nlm.nih.gov/pubchem/RDF/
+ftp_url="ftp://ftp.ncbi.nlm.nih.gov/pubchem/RDF/"
 
 # Create the download directory
 downloadpath="$localpath/download"
@@ -16,4 +18,4 @@ mkdir -p "$downloadpath"
 cd $downloadpath;
 
 # Download files
-aria2c -c -d $downloadpath $metalink_url
+aria2c -c -d $downloadpath $ftp_url

stages/02_build.sh

@@ -2,7 +2,7 @@
 
 set -euo pipefail
 
-# Script to convert RDF/XML to RDF HDT
+# Script to convert Turtle to RDF HDT
 
 # Get local path
 localpath=$(pwd)
@@ -24,7 +24,7 @@ brickpath="$localpath/brick"
 mkdir -p $brickpath
 echo "Brick path: $brickpath"
 
-base_uri="https://www.uniprot.org/"
+base_uri="http://rdf.ncbi.nlm.nih.gov/pubchem/"
 
 # Set TMPDIR on same filesystem
 export BUILD_TMPDIR=$buildpath/tmp
@@ -37,29 +37,25 @@ export TMPDIR=$BUILD_TMPDIR
 export buildpath_prestage=$buildpath/prestage
 mkdir -p $buildpath_prestage
 
-export buildpath brickpath base_uri
-find $downloadpath -type f -name '*.rdf.xz' | sort \
-	| grep -vFf <( cat <<'EOF' # remove empty files
-uniparc_patents.rdf.xz
-EOF
-	) \
+export downloadpath buildpath brickpath base_uri
+find $downloadpath -type f -name '*.ttl.gz' | sort \
 	| parallel -J ./parallel.prf --bar '
 		set -euo pipefail;
 
-		RDF=$buildpath/{/.};
-		RDF_HDT="$buildpath_prestage"/"$(basename "$RDF" .rdf).hdt";
+		RDF_HDT_DIR_REL="$(realpath -s --relative-to="$downloadpath" {})";
+		RDF_HDT="$buildpath_prestage"/"$RDF_HDT_DIR_REL"/"$(basename "$RDF" .rdf).hdt";
 
 		export RDF2HDTCAT_JAVA_OPTS="-Xmx24g";
 		if [ ! -s $RDF_HDT ]; then
 			echo "Processing {}"
-			xz -T1 -dk < {} \
-				| rapper --input rdfxml --output ntriples - "$base_uri" \
+			gzip -dk < {} \
+				| rapper --input turtle --output ntriples - "$base_uri" \
 				| rdf2hdtcat-parpipe $base_uri $RDF_HDT
 		fi
 		'
 
 find $downloadpath/ -maxdepth 1 \
-	-type f \! -name '*.rdf.xz' \
+	-type f \! -name '*.ttl.gz' \
 	-exec cp -v {} $brickpath/ \;
 
 mv -v $buildpath_prestage/* $brickpath/