From abde78b2eea0213822c707d8d1e375f60ec27cb5 Mon Sep 17 00:00:00 2001 From: Grzegorz Kostkowski Date: Wed, 18 Sep 2024 13:43:38 +0200 Subject: [PATCH 1/8] Add or modify Makefile recipes to enable regeneration of *.rdf files and retain expected *.ttl files content `rdfpipe` utility is now used to regenerate *.rdf files what results in the duplicated imports being discarded. Changes were made in Makefile file and affects generation of *.rdf artefacts (owl-core, owl-restrictions and shacl Makefile recipes) as well as conversion of these to TTL format (convert-rdf-to-turtle). The latter were needed to keep the same TTL output when using the new *.rdf files. Scope of the changes: * New generic convert-between-serialization-formats Makefile recipe * The existing owl-core, owl-restrictions and shacl Makefile recipes now include an RDF regeneration step * The existing convert-rdf-to-turtle Makefile recipe now uses the same namespaces that are used for XSLT transformation (test/ePO-default-config/namespaces.xml) to provide rdfpipe with prefixes so the output makes use of compact URIs. * As the reported issue occurs for EPO v4.2.0 which introduces some new prefixes, these prefixes have also been added to test/ePO-default-config/namespaces.xml so it would be possible to reproduce the fix. --- Makefile | 65 ++++++++++++++++++++++++-- scripts/get_namespaces.sh | 20 ++++++++ test/ePO-default-config/namespaces.xml | 11 +++-- 3 files changed, 89 insertions(+), 7 deletions(-) create mode 100755 scripts/get_namespaces.sh diff --git a/Makefile b/Makefile index 80df4822..4a51d21f 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,9 @@ TURTLE_FILELIST=$(shell ls ${ONTOLOGY_FOLDER_PATH}/*.ttl) # Widoco variables WIDOCO_RDF_INPUT_FILE_PATH?=test/reasoning-investigation/model-2020-12-16/ePO_restrictions.rdf WIDOCO_OUTPUT_FOLDER_PATH?=output/widoco +NAMESPACES_AS_RDFPIPE_ARGS=$(shell ${MODEL2OWL_FOLDER}/scripts/get_namespaces.sh) +RDF_XML_MIME_TYPE:='application/rdf+xml' +TURTLE_MIME_TYPE:='turtle' # download saxon library get-saxon: @@ -110,17 +113,37 @@ generate-convention-SVRL-report: #Example how to run transformation commands : # make owl-core XMI_INPUT_FILE_PATH=/home/mypc/work/model2owl/eNotice_CM.xml OUTPUT_FOLDER_PATH=./my-folder owl-core: - @java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/owl-core.xsl -o:${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}.rdf + @java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/owl-core.xsl \ + -o:${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}.tmp.rdf + @make convert-between-serialization-formats INPUT_FORMAT=${RDF_XML_MIME_TYPE} \ + OUTPUT_FORMAT=${RDF_XML_MIME_TYPE} \ + FILE_PATH=${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}.tmp.rdf \ + OUTPUT_FILE_PATH=${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}.rdf @echo Output owl core file: @ls -lh ${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}.rdf + @rm -f ${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}.tmp.rdf + owl-restrictions: - @java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/owl-restrictions.xsl -o:${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_restrictions.rdf + @java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/owl-restrictions.xsl \ + -o:${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_restrictions.tmp.rdf + @make convert-between-serialization-formats INPUT_FORMAT=${RDF_XML_MIME_TYPE} \ + OUTPUT_FORMAT=${RDF_XML_MIME_TYPE} \ + FILE_PATH=${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_restrictions.tmp.rdf \ + OUTPUT_FILE_PATH=${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_restrictions.rdf @echo Output owl restrictions file: @ls -lh ${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_restrictions.rdf + @rm -f ${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_restrictions.tmp.rdf + shacl: - @java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/shacl-shapes.xsl -o:${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_shapes.rdf + @java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/shacl-shapes.xsl \ + -o:${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_shapes.tmp.rdf + @make convert-between-serialization-formats INPUT_FORMAT=${RDF_XML_MIME_TYPE} \ + OUTPUT_FORMAT=${RDF_XML_MIME_TYPE} \ + FILE_PATH=${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_shapes.tmp.rdf \ + OUTPUT_FILE_PATH=${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_shapes.rdf @echo Output shacl file location: @ls -lh ${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_shapes.rdf + @rm -f ${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_shapes.tmp.rdf # Combine xmi UML files @@ -144,7 +167,12 @@ convert-rdf-to-turtle: @for FILE_PATH in ${RDF_FILELIST}; do \ echo Converting $${FILE_PATH} into Turtle; \ source model2owl-venv/bin/activate; \ - rdfpipe -i application/rdf+xml -o turtle $${FILE_PATH} > $${FILE_PATH%.*}.ttl; \ + make convert-between-serialization-formats \ + INPUT_FORMAT=${RDF_XML_MIME_TYPE} \ + OUTPUT_FORMAT=${TURTLE_MIME_TYPE} \ + FILE_PATH=$${FILE_PATH} \ + OUTPUT_FILE_PATH=$${FILE_PATH%.*}.ttl \ + USE_NAMESPACES=1; \ echo Input in RDF/XML format; \ echo $${FILE_PATH}; \ echo " ==> Output in Turtle format"; \ @@ -182,6 +210,35 @@ convert-rdf-to-rdf: echo " ==> Output in RDF/XML format"; \ ls -lh $${FILE_PATH%.*}.rdf; \ done + +# A generic recipe for converting RDF data from one serialization format to +# another. It can also be used to regenerate a file using the same format. +# +# Arguments: +# FILE_PATH: Input RDF file in any allowed serialization format +# OUTPUT_FILE_PATH: Path for the output file +# INPUT_FORMAT: a MIME type of the given input RDF file +# OUTPUT_FORMAT: a MIME type of any of the valid RDF serializations +# USE_NAMESPACES: optional; if non-empty then namespaces (from the namespaces.xml file). +# This can be used if the input (FILE_PATH) doesn't include +# namespaces we want to be applied (e.g. to have compact +# instead of full URIs in the output file). +# +# Supported MIME types: https://rdflib.readthedocs.io/en/7.0.0/plugin_serializers.html +# +# Example: +# make convert-between-serialization-formats +# INPUT_FORMAT='application/rdf+xml' +# OUTPUT_FORMAT='application/rdf+xml' +# FILE_PATH=output/ePO_core.tmp.rdf +# OUTPUT_FILE_PATH=output/ePO_core.rdf +# USE_NAMESPACES=1 +convert-between-serialization-formats: + @source model2owl-venv/bin/activate; \ + rdfpipe -i ${INPUT_FORMAT} -o ${OUTPUT_FORMAT} \ + $(if $(USE_NAMESPACES),${NAMESPACES_AS_RDFPIPE_ARGS}) \ + ${FILE_PATH} > ${OUTPUT_FILE_PATH} + # make validate-rdf-file FILE_TO_VALIDATE_PATH=./output/eFulfilment.rdf validate-rdf-file: @$(JENA_RIOT_TOOL) --validate $(FILE_TO_VALIDATE_PATH) diff --git a/scripts/get_namespaces.sh b/scripts/get_namespaces.sh new file mode 100755 index 00000000..7c26a703 --- /dev/null +++ b/scripts/get_namespaces.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Gets namespaces from namespaces.xml file and prepares argument +# list from them to be used with `rdfpipe` tool. +# Uses Saxon installed in the project main directory. + +PROJECT_DIR=$(dirname $(dirname $(realpath ${BASH_SOURCE[0]}))) +NAMESPACES_DIR=${PROJECT_DIR}/test/ePO-default-config + +cd ${NAMESPACES_DIR} +namespaces=$( + java -cp ../../saxon/saxon.jar net.sf.saxon.Query -s:namespaces.xml \ + -qs:'for $x in /*:prefixes/*:prefix return concat(string($x/@name), "=", string($x/@value))' \ + \!method=text +) +ns_args=$( \ + echo "$namespaces" | tr ' ' '\n' | awk '{printf("--ns='\''%s'\'' ", $0)}' +) + +echo "$ns_args" diff --git a/test/ePO-default-config/namespaces.xml b/test/ePO-default-config/namespaces.xml index 0cb964ac..42349dc4 100644 --- a/test/ePO-default-config/namespaces.xml +++ b/test/ePO-default-config/namespaces.xml @@ -12,13 +12,18 @@ + + - - + + + + + @@ -26,12 +31,12 @@ + - From f4a92472de60c7446026636e75999a6802008d07 Mon Sep 17 00:00:00 2001 From: Grzegorz Kostkowski Date: Thu, 19 Sep 2024 16:33:19 +0200 Subject: [PATCH 2/8] Parametrize get_namespaces.sh and set the required file path in the Makefile --- Makefile | 3 ++- scripts/get_namespaces.sh | 22 +++++++++++++++------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 4a51d21f..c5009dd2 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,8 @@ TURTLE_FILELIST=$(shell ls ${ONTOLOGY_FOLDER_PATH}/*.ttl) # Widoco variables WIDOCO_RDF_INPUT_FILE_PATH?=test/reasoning-investigation/model-2020-12-16/ePO_restrictions.rdf WIDOCO_OUTPUT_FOLDER_PATH?=output/widoco -NAMESPACES_AS_RDFPIPE_ARGS=$(shell ${MODEL2OWL_FOLDER}/scripts/get_namespaces.sh) +NAMESPACES_XML_FILE_PATH?=${MODEL2OWL_FOLDER}/test/ePO-default-config/namespaces.xml +NAMESPACES_AS_RDFPIPE_ARGS=$(shell ${MODEL2OWL_FOLDER}/scripts/get_namespaces.sh ${NAMESPACES_XML_FILE_PATH}) RDF_XML_MIME_TYPE:='application/rdf+xml' TURTLE_MIME_TYPE:='turtle' diff --git a/scripts/get_namespaces.sh b/scripts/get_namespaces.sh index 7c26a703..a08e37ec 100755 --- a/scripts/get_namespaces.sh +++ b/scripts/get_namespaces.sh @@ -1,15 +1,23 @@ #!/bin/bash # -# Gets namespaces from namespaces.xml file and prepares argument -# list from them to be used with `rdfpipe` tool. -# Uses Saxon installed in the project main directory. +# Gets namespaces from an XML file with the model2owl project namespaces +# (`namespaces.xml` by default) and prepares argument list from them to be +# used with `rdfpipe` tool. Uses Saxon installed in the project main directory. +# +# USAGE: get_namespaces.sh NAMESPACES_XML_FILE_PATH + +if [ -z "$1" ]; then + echo "ERROR: path to *.xml file with namespaces not given." + exit 1 +fi +namespaces_file_path="$1" -PROJECT_DIR=$(dirname $(dirname $(realpath ${BASH_SOURCE[0]}))) -NAMESPACES_DIR=${PROJECT_DIR}/test/ePO-default-config +namespaces_file_dir=$(dirname $(realpath $namespaces_file_path)) +namespaces_file_name=$(basename $namespaces_file_path) -cd ${NAMESPACES_DIR} +cd ${namespaces_file_dir} namespaces=$( - java -cp ../../saxon/saxon.jar net.sf.saxon.Query -s:namespaces.xml \ + java -cp ../../saxon/saxon.jar net.sf.saxon.Query -s:${namespaces_file_name} \ -qs:'for $x in /*:prefixes/*:prefix return concat(string($x/@name), "=", string($x/@value))' \ \!method=text ) From a8596c6e12fa1cecf8eb02aee3d6eb19cf2fe010 Mon Sep 17 00:00:00 2001 From: Grzegorz Kostkowski Date: Fri, 20 Sep 2024 17:30:07 +0200 Subject: [PATCH 3/8] Implement generation of enriched namespaces file, reorganize project to depend on the new file The new XML contains user namespaces (defined in namespaces.xml) and internal namespaces (such as core-shape). Both model2owl internals and get_namespaces.sh relies on that file and not on namespaces.xml. Moreover, model2owl internals now depends on a new template which generates namespaces declarations based on enriched file which is passed to the XSLT transformation engine as command line argument. For now, the path of the new XML file is kept in the Makefile, but this can be further improved in the future (if we come up with application internal XSL config or other solution). Scope of changes: * new XSL file for generation of the enriched namespaces XML file * unit test for the above XSL file * project codebase: * new function generating namespaces declarations and references to `$namespacePrefixes` replaced with `$internalNamespacePrefixes` * Makefile changes: * new recipe * existing 3 recipes generating *.rdf artefacts now include generation of the new XML file * required adjustments * Adjustments in the get_namespaces.sh script --- .gitignore | 1 + Makefile | 34 +++++++++++++----- scripts/get_namespaces.sh | 10 +++--- src/common/checkers.xsl | 8 ++--- src/common/utils.xsl | 18 +++++++++- src/owl-core.xsl | 6 ++-- src/owl-restrictions.xsl | 8 ++--- src/shacl-shapes.xsl | 10 ++---- src/xml/enriched-namespaces.xsl | 36 +++++++++++++++++++ .../test-xml/test-enriched-namespaces.xspec | 18 ++++++++++ 10 files changed, 113 insertions(+), 36 deletions(-) create mode 100644 src/xml/enriched-namespaces.xsl create mode 100644 test/unitTests/test-xml/test-enriched-namespaces.xspec diff --git a/.gitignore b/.gitignore index cc938d59..c2e099ce 100644 --- a/.gitignore +++ b/.gitignore @@ -325,3 +325,4 @@ lib64 pyvenv.cfg /build/ /node_modules/ +.temp diff --git a/Makefile b/Makefile index c5009dd2..a3739584 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ # Model2owl directory MODEL2OWL_FOLDER?=. +ABSOLUTE_MODEL2OWL_FOLDER?=$(shell realpath "${MODEL2OWL_FOLDER}") # rdflib version RDF_LIB_VERSION?=6.2.0 #Saxon path @@ -33,8 +34,10 @@ TURTLE_FILELIST=$(shell ls ${ONTOLOGY_FOLDER_PATH}/*.ttl) # Widoco variables WIDOCO_RDF_INPUT_FILE_PATH?=test/reasoning-investigation/model-2020-12-16/ePO_restrictions.rdf WIDOCO_OUTPUT_FOLDER_PATH?=output/widoco -NAMESPACES_XML_FILE_PATH?=${MODEL2OWL_FOLDER}/test/ePO-default-config/namespaces.xml -NAMESPACES_AS_RDFPIPE_ARGS=$(shell ${MODEL2OWL_FOLDER}/scripts/get_namespaces.sh ${NAMESPACES_XML_FILE_PATH}) +NAMESPACES_USER_XML_FILE_PATH?=${MODEL2OWL_FOLDER}/test/ePO-default-config/namespaces.xml +INTERM_FOLDER_PATH?=${ABSOLUTE_MODEL2OWL_FOLDER}/.temp +ENRICHED_NAMESPACES_XML_PATH:=${INTERM_FOLDER_PATH}/enriched-namespaces.xml +NAMESPACES_AS_RDFPIPE_ARGS=$(shell ${MODEL2OWL_FOLDER}/scripts/get_namespaces.sh ${ENRICHED_NAMESPACES_XML_PATH}) RDF_XML_MIME_TYPE:='application/rdf+xml' TURTLE_MIME_TYPE:='turtle' @@ -114,8 +117,10 @@ generate-convention-SVRL-report: #Example how to run transformation commands : # make owl-core XMI_INPUT_FILE_PATH=/home/mypc/work/model2owl/eNotice_CM.xml OUTPUT_FOLDER_PATH=./my-folder owl-core: + @make gen-enriched-ns-file @java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/owl-core.xsl \ - -o:${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}.tmp.rdf + -o:${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}.tmp.rdf \ + enrichedNamespacesPath="${ENRICHED_NAMESPACES_XML_PATH}" @make convert-between-serialization-formats INPUT_FORMAT=${RDF_XML_MIME_TYPE} \ OUTPUT_FORMAT=${RDF_XML_MIME_TYPE} \ FILE_PATH=${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}.tmp.rdf \ @@ -125,8 +130,10 @@ owl-core: @rm -f ${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}.tmp.rdf owl-restrictions: + @make gen-enriched-ns-file @java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/owl-restrictions.xsl \ - -o:${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_restrictions.tmp.rdf + -o:${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_restrictions.tmp.rdf \ + enrichedNamespacesPath="${ENRICHED_NAMESPACES_XML_PATH}" @make convert-between-serialization-formats INPUT_FORMAT=${RDF_XML_MIME_TYPE} \ OUTPUT_FORMAT=${RDF_XML_MIME_TYPE} \ FILE_PATH=${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_restrictions.tmp.rdf \ @@ -136,8 +143,10 @@ owl-restrictions: @rm -f ${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_restrictions.tmp.rdf shacl: + @make gen-enriched-ns-file @java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/shacl-shapes.xsl \ - -o:${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_shapes.tmp.rdf + -o:${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_shapes.tmp.rdf \ + enrichedNamespacesPath="${ENRICHED_NAMESPACES_XML_PATH}" @make convert-between-serialization-formats INPUT_FORMAT=${RDF_XML_MIME_TYPE} \ OUTPUT_FORMAT=${RDF_XML_MIME_TYPE} \ FILE_PATH=${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_shapes.tmp.rdf \ @@ -146,6 +155,12 @@ shacl: @ls -lh ${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_shapes.rdf @rm -f ${OUTPUT_FOLDER_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_shapes.tmp.rdf +# Generate enriched namespaces XML file which contains user namespaces (defined +# in namespaces.xml) and internal namespaces (such as core-shape) +gen-enriched-ns-file: + @mkdir -p ${INTERM_FOLDER_PATH} + @java -jar ${SAXON} -s:${NAMESPACES_USER_XML_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/xml/enriched-namespaces.xsl \ + -o:${ENRICHED_NAMESPACES_XML_PATH} # Combine xmi UML files # all files for combine should be in test/test-multi-xmi (or in XMI_MERGED_OUTPUT_FOLDER_PATH) @@ -220,10 +235,11 @@ convert-rdf-to-rdf: # OUTPUT_FILE_PATH: Path for the output file # INPUT_FORMAT: a MIME type of the given input RDF file # OUTPUT_FORMAT: a MIME type of any of the valid RDF serializations -# USE_NAMESPACES: optional; if non-empty then namespaces (from the namespaces.xml file). -# This can be used if the input (FILE_PATH) doesn't include -# namespaces we want to be applied (e.g. to have compact -# instead of full URIs in the output file). +# USE_NAMESPACES: optional; if non-empty then namespaces (from the +# enriched-namespaces.xml file). This can be used if the input +# (FILE_PATH) doesn't include namespaces we want to be applied +# (e.g. to have compact instead of full URIs in the output +# file). # # Supported MIME types: https://rdflib.readthedocs.io/en/7.0.0/plugin_serializers.html # diff --git a/scripts/get_namespaces.sh b/scripts/get_namespaces.sh index a08e37ec..f22d1d99 100755 --- a/scripts/get_namespaces.sh +++ b/scripts/get_namespaces.sh @@ -1,10 +1,12 @@ #!/bin/bash # -# Gets namespaces from an XML file with the model2owl project namespaces -# (`namespaces.xml` by default) and prepares argument list from them to be -# used with `rdfpipe` tool. Uses Saxon installed in the project main directory. +# Gets namespaces from an XML file with the model2owl project namespaces and +# prepares argument list from them to be used with `rdfpipe` tool. Uses Saxon +# installed in the project main directory. # # USAGE: get_namespaces.sh NAMESPACES_XML_FILE_PATH +PROJECT_DIR=$(dirname $(dirname $(realpath ${BASH_SOURCE[0]}))) +SAXON=${PROJECT_DIR}/saxon/saxon.jar if [ -z "$1" ]; then echo "ERROR: path to *.xml file with namespaces not given." @@ -17,7 +19,7 @@ namespaces_file_name=$(basename $namespaces_file_path) cd ${namespaces_file_dir} namespaces=$( - java -cp ../../saxon/saxon.jar net.sf.saxon.Query -s:${namespaces_file_name} \ + java -cp $SAXON net.sf.saxon.Query -s:${namespaces_file_name} \ -qs:'for $x in /*:prefixes/*:prefix return concat(string($x/@name), "=", string($x/@value))' \ \!method=text ) diff --git a/src/common/checkers.xsl b/src/common/checkers.xsl index b6ad192a..982063e7 100644 --- a/src/common/checkers.xsl +++ b/src/common/checkers.xsl @@ -512,15 +512,15 @@ - This function will check if a given list of namespaces are defined in - namespaces.xml file. If not all the namespaces were defined it will return a list with - those namespaces + This function will check if a given list of namespaces are + defined in enriched-namespaces.xml file. If not all the namespaces were + defined it will return a list with those namespaces + select="($internalNamespacePrefixes/*:prefixes/*:prefix/@name)"/> + + + Lookup a data-type in the xsd and rdf accepted data-type document (usually an external file with xsd and rdf data-types definitions) and return false or the data-type @@ -61,7 +68,7 @@ - + + + + + + diff --git a/src/owl-core.xsl b/src/owl-core.xsl index a2fb4041..f4d029be 100644 --- a/src/owl-core.xsl +++ b/src/owl-core.xsl @@ -46,9 +46,7 @@ - - - + @@ -63,7 +61,7 @@ - + diff --git a/src/owl-restrictions.xsl b/src/owl-restrictions.xsl index 22772ac9..fbcf6b2c 100644 --- a/src/owl-restrictions.xsl +++ b/src/owl-restrictions.xsl @@ -40,11 +40,7 @@ - - - - - + @@ -58,7 +54,7 @@ - + diff --git a/src/shacl-shapes.xsl b/src/shacl-shapes.xsl index 00c0349c..7892e87a 100644 --- a/src/shacl-shapes.xsl +++ b/src/shacl-shapes.xsl @@ -44,13 +44,7 @@ - - - - - - - + @@ -62,7 +56,7 @@ - + diff --git a/src/xml/enriched-namespaces.xsl b/src/xml/enriched-namespaces.xsl new file mode 100644 index 00000000..b3d9c88e --- /dev/null +++ b/src/xml/enriched-namespaces.xsl @@ -0,0 +1,36 @@ + + + + + + + + + A template for generating enriched namespaces XML file that + contains internal model2owl namespaces which are constructed + based on the model2owl configuration. + + + + + + + + + + + + + + + + + + diff --git a/test/unitTests/test-xml/test-enriched-namespaces.xspec b/test/unitTests/test-xml/test-enriched-namespaces.xspec new file mode 100644 index 00000000..ba778260 --- /dev/null +++ b/test/unitTests/test-xml/test-enriched-namespaces.xspec @@ -0,0 +1,18 @@ + + + + + + + + + + + From 63a9fb45817d8192064137af01f00e880e347e15 Mon Sep 17 00:00:00 2001 From: Grzegorz Kostkowski Date: Fri, 20 Sep 2024 18:05:07 +0200 Subject: [PATCH 4/8] Fix a test assertion and disable irrelevant check --- test/unitTests/test-xml/test-enriched-namespaces.xspec | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/unitTests/test-xml/test-enriched-namespaces.xspec b/test/unitTests/test-xml/test-enriched-namespaces.xspec index ba778260..ff422095 100644 --- a/test/unitTests/test-xml/test-enriched-namespaces.xspec +++ b/test/unitTests/test-xml/test-enriched-namespaces.xspec @@ -4,6 +4,11 @@ xmlns:xs="http://www.w3.org/2001/XMLSchema" stylesheet="../../../src/xml/enriched-namespaces.xsl"> + + + + test="*:prefixes/*:prefix[@name = 'core-shape']/@value = 'http://data.europa.eu/a4g/data-shape#'"/> From ce129be729bc7420dbbb00afd7ab743402314a98 Mon Sep 17 00:00:00 2001 From: Grzegorz Kostkowski Date: Fri, 20 Sep 2024 19:20:05 +0200 Subject: [PATCH 5/8] Pass required parameter to the testing process --- Makefile | 2 +- pom.xml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a3739584..75a511d3 100644 --- a/Makefile +++ b/Makefile @@ -73,7 +73,7 @@ install: get-saxon get-rdflib get-widoco ############################ Main tasks ############################################## # Run unit_tests unit-tests: - @mvn install + @mvn install -Dsaxon.options.enrichedNamespacesPath=${ENRICHED_NAMESPACES_XML_PATH} create-virtual-env: @python -m venv model2owl-venv diff --git a/pom.xml b/pom.xml index 61bb1c03..d6e8e564 100644 --- a/pom.xml +++ b/pom.xml @@ -25,6 +25,7 @@ true false + enrichedNamespacesPath=${saxon.options.enrichedNamespacesPath} From 1de57428f4e60b8802ff126c39b6af3a317e9d2e Mon Sep 17 00:00:00 2001 From: Grzegorz Kostkowski Date: Mon, 23 Sep 2024 11:24:19 +0200 Subject: [PATCH 6/8] Accomodate for the new ns file, use make in the github action --- .github/workflows/unit-tests.yml | 2 +- Makefile | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index e81ec8c6..d90f6fb7 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -20,7 +20,7 @@ jobs: with: fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis - name: Build and Run Tests - run: mvn clean install + run: make install unit-tests - name: current dir run: pwd - name: current dir diff --git a/Makefile b/Makefile index 75a511d3..792f1f63 100644 --- a/Makefile +++ b/Makefile @@ -68,13 +68,17 @@ get-widoco: ###################################################################################### # Download, install saxon, xspec, rdflib and other dependencies ###################################################################################### -install: get-saxon get-rdflib get-widoco +install: get-saxon create-virtual-env get-rdflib get-widoco ############################ Main tasks ############################################## # Run unit_tests unit-tests: + @make test-prerequisites @mvn install -Dsaxon.options.enrichedNamespacesPath=${ENRICHED_NAMESPACES_XML_PATH} +test-prerequisites: + @make gen-enriched-ns-file + create-virtual-env: @python -m venv model2owl-venv @@ -84,6 +88,7 @@ create-virtual-env: # make generate-glossary XMI_INPUT_FILE_PATH=/home/mypc/work/model2owl/eNotice_CM.xml OUTPUT_GLOSSARY_PATH=/home/mypc/work/model2owl/glossary generate-glossary: @mkdir -p "${OUTPUT_GLOSSARY_PATH}" + @make gen-enriched-ns-file @echo Input file path: ${XMI_INPUT_FILE_PATH} @echo Input file name: ${XMI_INPUT_FILENAME_WITHOUT_EXTENSION} @cp -rf ./src/static "${OUTPUT_GLOSSARY_PATH}" @@ -95,6 +100,7 @@ generate-glossary: generate-convention-report: @mkdir -p "${OUTPUT_CONVENTION_REPORT_PATH}" + @make gen-enriched-ns-file @echo Input file path: ${XMI_INPUT_FILE_PATH} @echo Input file name: ${XMI_INPUT_FILENAME_WITHOUT_EXTENSION} @cp -rf ./src/static "${OUTPUT_CONVENTION_REPORT_PATH}" @@ -106,6 +112,7 @@ generate-convention-report: generate-convention-SVRL-report: @mkdir -p "${OUTPUT_CONVENTION_REPORT_PATH}" + @make gen-enriched-ns-file @echo Input file path: ${XMI_INPUT_FILE_PATH} @echo Input file name: ${XMI_INPUT_FILENAME_WITHOUT_EXTENSION} @cp -rf ./src/static "${OUTPUT_CONVENTION_REPORT_PATH}" @@ -180,6 +187,7 @@ merge-xmi: # make convert-to-turtle ONTOLOGY_FOLDER_PATH=./my-folder # ONTOLOGY_FOLDER_PATH is the the path to the folder containing .rdf files for converting to turtle or .ttl files to convert to rdf convert-rdf-to-turtle: + @make gen-enriched-ns-file @for FILE_PATH in ${RDF_FILELIST}; do \ echo Converting $${FILE_PATH} into Turtle; \ source model2owl-venv/bin/activate; \ From 230908765b751f0f1e37027f1013337c7b078151 Mon Sep 17 00:00:00 2001 From: Grzegorz Kostkowski Date: Mon, 23 Sep 2024 11:46:06 +0200 Subject: [PATCH 7/8] Remove fixed declaration of derived internal ns from namespaces file, pass new namespaces file as a parameter for the rest of xslt transformation recipes in the project --- Makefile | 15 ++++++++++++--- test/ePO-default-config/namespaces.xml | 1 - 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 792f1f63..f2ed45d6 100644 --- a/Makefile +++ b/Makefile @@ -92,7 +92,10 @@ generate-glossary: @echo Input file path: ${XMI_INPUT_FILE_PATH} @echo Input file name: ${XMI_INPUT_FILENAME_WITHOUT_EXTENSION} @cp -rf ./src/static "${OUTPUT_GLOSSARY_PATH}" - @java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/html-model-glossary.xsl -o:${OUTPUT_GLOSSARY_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_glossary.html + @java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} \ + -xsl:${MODEL2OWL_FOLDER}/src/html-model-glossary.xsl \ + -o:${OUTPUT_GLOSSARY_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_glossary.html \ + enrichedNamespacesPath="${ENRICHED_NAMESPACES_XML_PATH}" @echo The glossary is located at the following location: @echo @ls -lh ${OUTPUT_GLOSSARY_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_glossary.html @@ -104,7 +107,10 @@ generate-convention-report: @echo Input file path: ${XMI_INPUT_FILE_PATH} @echo Input file name: ${XMI_INPUT_FILENAME_WITHOUT_EXTENSION} @cp -rf ./src/static "${OUTPUT_CONVENTION_REPORT_PATH}" - @java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/html-conventions-report.xsl -o:${OUTPUT_CONVENTION_REPORT_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_convention_report.html + @java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} \ + -xsl:${MODEL2OWL_FOLDER}/src/html-conventions-report.xsl \ + -o:${OUTPUT_CONVENTION_REPORT_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_convention_report.html \ + enrichedNamespacesPath="${ENRICHED_NAMESPACES_XML_PATH}" @echo The convention report is located at the following location: @echo @ls -lh ${OUTPUT_CONVENTION_REPORT_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_convention_report.html @@ -116,7 +122,10 @@ generate-convention-SVRL-report: @echo Input file path: ${XMI_INPUT_FILE_PATH} @echo Input file name: ${XMI_INPUT_FILENAME_WITHOUT_EXTENSION} @cp -rf ./src/static "${OUTPUT_CONVENTION_REPORT_PATH}" - @java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/svrl-conventions-report.xsl -o:${OUTPUT_CONVENTION_REPORT_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_convention_svrl_report.xml + @java -jar ${SAXON} -s:${XMI_INPUT_FILE_PATH} \ + -xsl:${MODEL2OWL_FOLDER}/src/svrl-conventions-report.xsl \ + -o:${OUTPUT_CONVENTION_REPORT_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_convention_svrl_report.xml \ + enrichedNamespacesPath="${ENRICHED_NAMESPACES_XML_PATH}" @echo The convention report is located at the following location: @echo @ls -lh ${OUTPUT_CONVENTION_REPORT_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_convention_svrl_report.xml diff --git a/test/ePO-default-config/namespaces.xml b/test/ePO-default-config/namespaces.xml index 42349dc4..4f44ddae 100644 --- a/test/ePO-default-config/namespaces.xml +++ b/test/ePO-default-config/namespaces.xml @@ -23,7 +23,6 @@ - From 580bccc029c759afcd741f7b816d6046d8f8d4e7 Mon Sep 17 00:00:00 2001 From: Grzegorz Kostkowski Date: Tue, 24 Sep 2024 09:44:37 +0200 Subject: [PATCH 8/8] Update and refine docstrings in the Makefile Document use of the new `NAMESPACES_USER_XML_FILE_PATH` variable. Any recipe without a docstring (comment documenting its use) that uses the new variable now got a description. --- Makefile | 47 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index f2ed45d6..4b64a0c8 100644 --- a/Makefile +++ b/Makefile @@ -76,6 +76,11 @@ unit-tests: @make test-prerequisites @mvn install -Dsaxon.options.enrichedNamespacesPath=${ENRICHED_NAMESPACES_XML_PATH} +# Actions required in order to setup the environment for testing purposes. +# Usage (`[]` denotes an optional argument; if omited, default value will be used): +# make test-prerequisites [NAMESPACES_USER_XML_FILE_PATH=/path/to/namespaces.xml] +# where: +# NAMESPACES_USER_XML_FILE_PATH: path to the *.xml file provided by a user test-prerequisites: @make gen-enriched-ns-file @@ -84,6 +89,13 @@ create-virtual-env: # Generate the glossary from an input file +# Usage (`[]` denotes an optional argument; if omited, default value will be used): +# make generate-glossary [XMI_INPUT_FILE_PATH=/path/to/cm.xmi] +# [OUTPUT_GLOSSARY_PATH=/output/directory] +# [NAMESPACES_USER_XML_FILE_PATH=/path/to/namespaces.xml] +# where: +# NAMESPACES_USER_XML_FILE_PATH: path to the *.xml file provided by a user +# # Example when not using the default variables # make generate-glossary XMI_INPUT_FILE_PATH=/home/mypc/work/model2owl/eNotice_CM.xml OUTPUT_GLOSSARY_PATH=/home/mypc/work/model2owl/glossary generate-glossary: @@ -101,6 +113,14 @@ generate-glossary: @ls -lh ${OUTPUT_GLOSSARY_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_glossary.html @echo +# Usage of the convention report generation recipes (generate-convention-report | generate-convention-SVRL-report). +# `[]` denotes an optional argument; if omited, default value will be used: +# make (generate-convention-report | generate-convention-SVRL-report) +# [XMI_INPUT_FILE_PATH=/path/to/cm.xmi] +# [OUTPUT_CONVENTION_REPORT_PATH=/output/directory] +# [NAMESPACES_USER_XML_FILE_PATH=/path/to/namespaces.xml] +# where: +# NAMESPACES_USER_XML_FILE_PATH: path to the *.xml file provided by a user generate-convention-report: @mkdir -p "${OUTPUT_CONVENTION_REPORT_PATH}" @make gen-enriched-ns-file @@ -130,7 +150,17 @@ generate-convention-SVRL-report: @echo @ls -lh ${OUTPUT_CONVENTION_REPORT_PATH}/${XMI_INPUT_FILENAME_WITHOUT_EXTENSION}_convention_svrl_report.xml @echo -#Example how to run transformation commands : + + +# Usage of the transformation recipes (owl-core | owl-restrictions | shacl). +# `[]` denotes an optional argument; if omited, default value will be used: +# make (owl-core | owl-restrictions | shacl) [XMI_INPUT_FILE_PATH=/path/to/cm.xmi] +# [OUTPUT_FOLDER_PATH=/output/directory] +# [NAMESPACES_USER_XML_FILE_PATH=/path/to/namespaces.xml] +# where: +# NAMESPACES_USER_XML_FILE_PATH: path to the *.xml file provided by a user +# +# Example: # make owl-core XMI_INPUT_FILE_PATH=/home/mypc/work/model2owl/eNotice_CM.xml OUTPUT_FOLDER_PATH=./my-folder owl-core: @make gen-enriched-ns-file @@ -173,6 +203,10 @@ shacl: # Generate enriched namespaces XML file which contains user namespaces (defined # in namespaces.xml) and internal namespaces (such as core-shape) +# Usage (`[]` denotes an optional argument; if omited, default value will be used): +# make gen-enriched-ns-file [NAMESPACES_USER_XML_FILE_PATH=/path/to/namespaces.xml] +# where: +# NAMESPACES_USER_XML_FILE_PATH: path to the *.xml file provided by a user gen-enriched-ns-file: @mkdir -p ${INTERM_FOLDER_PATH} @java -jar ${SAXON} -s:${NAMESPACES_USER_XML_FILE_PATH} -xsl:${MODEL2OWL_FOLDER}/src/xml/enriched-namespaces.xsl \ @@ -192,9 +226,14 @@ merge-xmi: -#Example how to run converting commands : -# make convert-to-turtle ONTOLOGY_FOLDER_PATH=./my-folder -# ONTOLOGY_FOLDER_PATH is the the path to the folder containing .rdf files for converting to turtle or .ttl files to convert to rdf +# Example how to run converting commands (`[]` denotes an optional argument; +# if omited, default value will be used): +# make convert-to-turtle [ONTOLOGY_FOLDER_PATH=./my-folder] +# [NAMESPACES_USER_XML_FILE_PATH=/path/to/namespaces.xml] +# where: +# ONTOLOGY_FOLDER_PATH: the path to the folder containing .rdf files for +# converting to turtle or .ttl files to convert to rdf +# NAMESPACES_USER_XML_FILE_PATH: path to the *.xml file provided by a user convert-rdf-to-turtle: @make gen-enriched-ns-file @for FILE_PATH in ${RDF_FILELIST}; do \