update MASPAWIO harvest

collection/scripts/maspawio-harvest.py

@@ -3,14 +3,16 @@
 """
 Purpose: Standalone script to generate individual JSON-LD files
          and a master RDF document, live, from CSW endpoint
-         
+
 Usage:   python maspawio-harvest.py
 
 Output:  saves a new JSON-LD file, for each catalogue record 
          exposed through the OGC:CSW service, and also a
          .RDF resource file.
          
-Requires: Python 3.x
+Requires: - Python 3.x
+          - must first execute:
+            python -m pip install -r requirements-maspawio-harvest.txt
 
 Notes:
 
@@ -24,12 +26,12 @@
 # define common variables
 CSW_ENDPOINT = "http://maspawio.net/catalogue/csw"
 CSW_ENDPOINT_TIMEOUT = 60 #seconds
-PATH_TO_DATA_FOLDER = "./data-maspawio/"
+PATH_TO_DATA_FOLDER = "../tempHosting/data-maspawio/"
 NEW_RDF_FILENAME = "maspawio-catalogue.rdf"
 HOSTNAME = "http://maspawio.net"
 LOGFILE = "maspawio-harvest.log"
-SHORTNAME = "maspawio" #must be hyphen
-ID_URL_BASE = "https://raw.githubusercontent.com/iodepo/odis-arch/schema-dev-jm/code/notebooks/Exploration/data-maspawio/"
+SHORTNAME = "maspawio" #must not contain spaces, but can contain hyphen
+ID_URL_BASE = "https://raw.githubusercontent.com/iodepo/odis-arch/collection/tempHosting/data-maspawio/"
 
 """
 #########################
@@ -42,8 +44,6 @@
 import os, sys, io, uuid
 from owslib.csw import CatalogueServiceWeb
 from owslib.fes import SortBy, SortProperty
-import ssl
-import pandas as pd
 import kglab
 import logging
 
@@ -52,13 +52,6 @@
                     format="%(asctime)s;%(levelname)s;%(message)s",  
                     datefmt="%Y-%m-%d %H:%M", filemode = "w")
 
-# generate a Context for each connection
-# disable SSL for now
-
-ctx = ssl.create_default_context()
-ctx.check_hostname = False
-ctx.verify_mode = ssl.CERT_NONE
-
 # prepare namespace
 
 namespaces = {
@@ -141,7 +134,7 @@
             #id
             id = csw.records[rec].identifier
             logging.info("record id: %s", id)
-
+            
             #description
             description = csw.records[rec].identification.abstract #ISO
             #description = csw.records[rec].abstract #DublinCore
@@ -159,7 +152,17 @@
             maxx = csw.records[rec].identification.bbox.maxx
             maxy = csw.records[rec].identification.bbox.maxy
 
-            poly = str("""POLYGON(({} {}, {} {}, {} {}, {} {}, {} {}))""".format(minx, miny, minx, maxy, maxx, maxy, maxx, miny, minx, miny))
+            #poly = str("""POLYGON(({} {}, {} {}, {} {}, {} {}, {} {}))""".format(minx, miny, minx, maxy, maxx, maxy, maxx, miny, minx, miny))
+
+            #schema.org expects lat long (Y X) coordinate order
+            boxCoords = str("""{} {} {} {}""".format(miny, minx, maxy, maxx))
+            print("    GeoShape:Box: " + boxCoords)
+            spatialCov = {}
+            spatialCov["@type"] = "https://schema.org/Place"
+            geo = {}
+            geo["@type"] = "https://schema.org/GeoShape"
+            geo["https://schema.org/box"] = boxCoords 
+            spatialCov["https://schema.org/geo"] = geo
 
             data = {}
 
@@ -177,19 +180,7 @@
             data["https://schema.org/description"] = description
             data["https://schema.org/url"] = url
 
-            aswkt = {}
-            aswkt["@type"] = "http://www.opengis.net/ont/geosparql#wktLiteral"
-            aswkt["@value"] = poly
-
-            crs = {}
-            crs["@id"] = "http://www.opengis.net/def/crs/OGC/1.3/CRS84"
-
-            hg = {}
-            hg["@type"] = "http://www.opengis.net/ont/sf#Polygon" 
-            hg["http://www.opengis.net/ont/geosparql#asWKT"] = aswkt
-            hg["http://www.opengis.net/ont/geosparql#crs"] = crs
-
-            data["http://www.opengis.net/ont/geosparql#hasGeometry"] = hg
+            data["https://schema.org/spatialCoverage"] = spatialCov 
 
             # keyword(s) loop
             k = ""
@@ -212,7 +203,7 @@
             k_list = k.split(",")
             data["https://schema.org/keywords"] = k_list
 
-            context = {"@vocab": "https://schema.org/", "geosparql": "http://www.opengis.net/ont/geosparql#"}
+            context = {"@vocab": "https://schema.org/"}
             compacted = jsonld.compact(data, context)
 
             # need sha hash for the "compacted" var and then also generate the prov for this record.

collection/scripts/requirements-maspawio-harvest.txt

@@ -0,0 +1,3 @@
+pyld
+owslib
+kglab

collection/tempHosting/data-maspawio/maspawio-0030a210-c413-11e9-ac48-0a735f7a740c.json

@@ -1,20 +1,9 @@
 {
     "@context": {
-        "@vocab": "https://schema.org/",
-        "geosparql": "http://www.opengis.net/ont/geosparql#"
+        "@vocab": "https://schema.org/"
     },
-    "@id": "https://raw.githubusercontent.com/iodepo/odis-arch/schema-dev-jm/code/notebooks/Exploration/data-maspawio/maspawio-0030a210-c413-11e9-ac48-0a735f7a740c.json",
+    "@id": "https://raw.githubusercontent.com/iodepo/odis-arch/collection/tempHosting/data-maspawio/maspawio-0030a210-c413-11e9-ac48-0a735f7a740c.json",
     "@type": "Dataset",
-    "geosparql:hasGeometry": {
-        "@type": "http://www.opengis.net/ont/sf#Polygon",
-        "geosparql:asWKT": {
-            "@type": "geosparql:wktLiteral",
-            "@value": "POLYGON((39.3 -4.649999999999997, 39.3 -4.19, 39.709999999999994 -4.19, 39.709999999999994 -4.649999999999997, 39.3 -4.649999999999997))"
-        },
-        "geosparql:crs": {
-            "@id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84"
-        }
-    },
     "description": "This data shows high-resolution maps on fishing behavior and target resources of artisanal fishers in Gazi Kenya. The resource was developed using participatory mapping of fishing grounds using a handheld GPS mounted on selected fishing vessels. To access this resource, please contact innocentke@gmail.com or jmbugua@cordioea.net",
     "keywords": [
         "fishing",
@@ -23,5 +12,12 @@
         "Kenya"
     ],
     "name": "Fishing intensity Gazi",
+    "spatialCoverage": {
+        "@type": "Place",
+        "geo": {
+            "@type": "GeoShape",
+            "box": "-4.649999999999997 39.3 -4.19 39.709999999999994"
+        }
+    },
     "url": "http://maspawio.net/layers/geonode%3Agazi_fishing_intensity"
 }

collection/tempHosting/data-maspawio/maspawio-0120a498-7204-11e8-9264-0a735f7a740c.json

@@ -1,22 +1,18 @@
 {
     "@context": {
-        "@vocab": "https://schema.org/",
-        "geosparql": "http://www.opengis.net/ont/geosparql#"
+        "@vocab": "https://schema.org/"
     },
-    "@id": "https://raw.githubusercontent.com/iodepo/odis-arch/schema-dev-jm/code/notebooks/Exploration/data-maspawio/maspawio-0120a498-7204-11e8-9264-0a735f7a740c.json",
+    "@id": "https://raw.githubusercontent.com/iodepo/odis-arch/collection/tempHosting/data-maspawio/maspawio-0120a498-7204-11e8-9264-0a735f7a740c.json",
     "@type": "Dataset",
-    "geosparql:hasGeometry": {
-        "@type": "http://www.opengis.net/ont/sf#Polygon",
-        "geosparql:asWKT": {
-            "@type": "geosparql:wktLiteral",
-            "@value": "POLYGON((32.867106640559 -27.0173701471146, 32.867106640559 12.9389764622939, 72.7374771162759 12.9389764622939, 72.7374771162759 -27.0173701471146, 32.867106640559 -27.0173701471146))"
-        },
-        "geosparql:crs": {
-            "@id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84"
-        }
-    },
     "description": "No abstract provided",
     "keywords": "",
     "name": "wio_reef_by_intergrated_local",
+    "spatialCoverage": {
+        "@type": "Place",
+        "geo": {
+            "@type": "GeoShape",
+            "box": "-27.0173701471146 32.867106640559 12.9389764622939 72.7374771162759"
+        }
+    },
     "url": "http://maspawio.net/layers/geonode%3Awio_reef_by_intergrated_loacal"
 }

collection/tempHosting/data-maspawio/maspawio-028adc56-c413-11e9-8a24-0a735f7a740c.json

@@ -1,20 +1,9 @@
 {
     "@context": {
-        "@vocab": "https://schema.org/",
-        "geosparql": "http://www.opengis.net/ont/geosparql#"
+        "@vocab": "https://schema.org/"
     },
-    "@id": "https://raw.githubusercontent.com/iodepo/odis-arch/schema-dev-jm/code/notebooks/Exploration/data-maspawio/maspawio-028adc56-c413-11e9-8a24-0a735f7a740c.json",
+    "@id": "https://raw.githubusercontent.com/iodepo/odis-arch/collection/tempHosting/data-maspawio/maspawio-028adc56-c413-11e9-8a24-0a735f7a740c.json",
     "@type": "Dataset",
-    "geosparql:hasGeometry": {
-        "@type": "http://www.opengis.net/ont/sf#Polygon",
-        "geosparql:asWKT": {
-            "@type": "geosparql:wktLiteral",
-            "@value": "POLYGON((40.5 -2.899999999999998, 40.5 -2.4800000000000018, 40.91 -2.4800000000000018, 40.91 -2.899999999999998, 40.5 -2.899999999999998))"
-        },
-        "geosparql:crs": {
-            "@id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84"
-        }
-    },
     "description": "This data shows high-resolution maps on fishing behavior and target resources of artisanal fishers in Kipini Kenya. The resource was developed using participatory mapping of fishing grounds using a handheld GPS mounted on selected fishing vessels. To access this resource, please contact innocentke@gmail.com or jmbugua@cordioea.net",
     "keywords": [
         "fishing",
@@ -23,5 +12,12 @@
         "Kenya"
     ],
     "name": "Fishing intensity Kipini",
+    "spatialCoverage": {
+        "@type": "Place",
+        "geo": {
+            "@type": "GeoShape",
+            "box": "-2.899999999999998 40.5 -2.4800000000000018 40.91"
+        }
+    },
     "url": "http://maspawio.net/layers/geonode%3Akipini_fishing_intensity"
 }

collection/tempHosting/data-maspawio/maspawio-032a1d26-d4e5-11e7-9524-0a735f7a740c.json

@@ -1,20 +1,9 @@
 {
     "@context": {
-        "@vocab": "https://schema.org/",
-        "geosparql": "http://www.opengis.net/ont/geosparql#"
+        "@vocab": "https://schema.org/"
     },
-    "@id": "https://raw.githubusercontent.com/iodepo/odis-arch/schema-dev-jm/code/notebooks/Exploration/data-maspawio/maspawio-032a1d26-d4e5-11e7-9524-0a735f7a740c.json",
+    "@id": "https://raw.githubusercontent.com/iodepo/odis-arch/collection/tempHosting/data-maspawio/maspawio-032a1d26-d4e5-11e7-9524-0a735f7a740c.json",
     "@type": "Dataset",
-    "geosparql:hasGeometry": {
-        "@type": "http://www.opengis.net/ont/sf#Polygon",
-        "geosparql:asWKT": {
-            "@type": "geosparql:wktLiteral",
-            "@value": "POLYGON((40.458759594707196 -11.274668553927912, 40.458759594707196 -10.58445333930988, 40.74266261873406 -10.58445333930988, 40.74266261873406 -11.274668553927912, 40.458759594707196 -11.274668553927912))"
-        },
-        "geosparql:crs": {
-            "@id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84"
-        }
-    },
     "description": "This raster data illustrates site/project level evaluation of marine habitats of Lalane and Quiwia in the Quirimbas Archipelago, Mozambique.\nThe output was produced by an independent project, but using layers collated for MOZALINK module activities.\nData collection and processing involved supervised image classification based Landsat-8 OLI Sensor data at 30-meter resolution utilizing the Blue, Green and Red Spectral bands for interpretation. \nGround control points were collected through collaborative work between Coastal Ocean Research for Development in the Indian Ocean (CORDIO EA), AMA and the Conselho Comunitário de Pesca (CCPs) of Quiwia and Lalane. \nAll image processing/analysis was conducted using QGIS Semi-Automatic Classification plugin to derive a benthic habitat data layer.",
     "keywords": [
         "benthic habitat",
@@ -24,5 +13,12 @@
         "Mozambique"
     ],
     "name": "Quiwia Lalane habitat classification",
+    "spatialCoverage": {
+        "@type": "Place",
+        "geo": {
+            "@type": "GeoShape",
+            "box": "-11.274668553927912 40.458759594707196 -10.58445333930988 40.74266261873406"
+        }
+    },
     "url": "http://maspawio.net/layers/geonode%3Aquiwia_lalane_habitat_classification"
 }

collection/tempHosting/data-maspawio/maspawio-03be4f3c-14c1-11ea-98e8-0a735f7a740c.json

@@ -1,22 +1,18 @@
 {
     "@context": {
-        "@vocab": "https://schema.org/",
-        "geosparql": "http://www.opengis.net/ont/geosparql#"
+        "@vocab": "https://schema.org/"
     },
-    "@id": "https://raw.githubusercontent.com/iodepo/odis-arch/schema-dev-jm/code/notebooks/Exploration/data-maspawio/maspawio-03be4f3c-14c1-11ea-98e8-0a735f7a740c.json",
+    "@id": "https://raw.githubusercontent.com/iodepo/odis-arch/collection/tempHosting/data-maspawio/maspawio-03be4f3c-14c1-11ea-98e8-0a735f7a740c.json",
     "@type": "Dataset",
-    "geosparql:hasGeometry": {
-        "@type": "http://www.opengis.net/ont/sf#Polygon",
-        "geosparql:asWKT": {
-            "@type": "geosparql:wktLiteral",
-            "@value": "POLYGON((39.2188816911614 -4.71986163841819, 39.2188816911614 -4.11922536090651, 39.6233411109311 -4.11922536090651, 39.6233411109311 -4.71986163841819, 39.2188816911614 -4.71986163841819))"
-        },
-        "geosparql:crs": {
-            "@id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84"
-        }
-    },
     "description": "Flood prone location kwale County is a vector file showing areas below one-meter elevation. The data was derived by CORDIO from the Space Shuttle Radar Topography Mission (SRTM) 30 meter DEM data. \nThe layer reflects places that are prone to coastal flood projection according to the selected elevation dataset.",
     "keywords": "elevation",
     "name": "Flood prone location kwale County",
+    "spatialCoverage": {
+        "@type": "Place",
+        "geo": {
+            "@type": "GeoShape",
+            "box": "-4.71986163841819 39.2188816911614 -4.11922536090651 39.6233411109311"
+        }
+    },
     "url": "http://maspawio.net/layers/geonode%3Alocation_below_one_meter_elevation_kwale_counnty"
 }

collection/tempHosting/data-maspawio/maspawio-0416d4ca-e26a-11e8-846f-0a735f7a740c.json

@@ -1,22 +1,18 @@
 {
     "@context": {
-        "@vocab": "https://schema.org/",
-        "geosparql": "http://www.opengis.net/ont/geosparql#"
+        "@vocab": "https://schema.org/"
     },
-    "@id": "https://raw.githubusercontent.com/iodepo/odis-arch/schema-dev-jm/code/notebooks/Exploration/data-maspawio/maspawio-0416d4ca-e26a-11e8-846f-0a735f7a740c.json",
+    "@id": "https://raw.githubusercontent.com/iodepo/odis-arch/collection/tempHosting/data-maspawio/maspawio-0416d4ca-e26a-11e8-846f-0a735f7a740c.json",
     "@type": "Dataset",
-    "geosparql:hasGeometry": {
-        "@type": "http://www.opengis.net/ont/sf#Polygon",
-        "geosparql:asWKT": {
-            "@type": "geosparql:wktLiteral",
-            "@value": "POLYGON((-180 -90, -180 90, 180 90, 180 -90, -180 -90))"
-        },
-        "geosparql:crs": {
-            "@id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84"
-        }
-    },
     "description": "No abstract provided",
     "keywords": "",
     "name": "MS_WIO_Fish_Diversity_survey_sites_2018.txt",
+    "spatialCoverage": {
+        "@type": "Place",
+        "geo": {
+            "@type": "GeoShape",
+            "box": "-90 -180 90 180"
+        }
+    },
     "url": "http://maspawio.net/documents/360"
 }

collection/tempHosting/data-maspawio/maspawio-047cac4c-c413-11e9-95bc-0a735f7a740c.json

@@ -1,20 +1,9 @@
 {
     "@context": {
-        "@vocab": "https://schema.org/",
-        "geosparql": "http://www.opengis.net/ont/geosparql#"
+        "@vocab": "https://schema.org/"
     },
-    "@id": "https://raw.githubusercontent.com/iodepo/odis-arch/schema-dev-jm/code/notebooks/Exploration/data-maspawio/maspawio-047cac4c-c413-11e9-95bc-0a735f7a740c.json",
+    "@id": "https://raw.githubusercontent.com/iodepo/odis-arch/collection/tempHosting/data-maspawio/maspawio-047cac4c-c413-11e9-95bc-0a735f7a740c.json",
     "@type": "Dataset",
-    "geosparql:hasGeometry": {
-        "@type": "http://www.opengis.net/ont/sf#Polygon",
-        "geosparql:asWKT": {
-            "@type": "geosparql:wktLiteral",
-            "@value": "POLYGON((39.15 -4.8, 39.15 -4.49, 39.54 -4.49, 39.54 -4.8, 39.15 -4.8))"
-        },
-        "geosparql:crs": {
-            "@id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84"
-        }
-    },
     "description": "This data shows high-resolution maps on fishing behavior and target resources of artisanal fishers in Vanga Jimbo Kenya. The resource was developed using participatory mapping of fishing grounds using a handheld GPS mounted on selected fishing vessels. To access this resource, please contact innocentke@gmail.com or jmbugua@cordioea.net",
     "keywords": [
         "fishing",
@@ -23,5 +12,12 @@
         "Kenya"
     ],
     "name": "Fishing intensity Vanga Jimbo",
+    "spatialCoverage": {
+        "@type": "Place",
+        "geo": {
+            "@type": "GeoShape",
+            "box": "-4.8 39.15 -4.49 39.54"
+        }
+    },
     "url": "http://maspawio.net/layers/geonode%3Avanga_jimbo__fishing_intensity"
 }