build.qmd

---
title:  Workflow for building APD
---

```{r}
library(dplyr)
library(tidyr)
library(readr)
library(stringr)
library(rdflib)
source("R/convert_to_triples.R")
```

Load data and create triples
```{r}
annotation_properties_csv <- read_csv("data/APD_annotation_properties.csv", show_col_types = FALSE)
traits_csv <- read_csv("data/APD_traits_input.csv", show_col_types = FALSE) %>%
  mutate(Entity = paste0("https://w3id.org/APD/traits/", identifier))
glossary_csv <- read_csv("data/APD_glossary.csv", show_col_types = FALSE) %>%
  mutate(Entity = paste0("https://w3id.org/APD/glossary/", identifier))
published_classes_csv <- read_csv("data/published_classes.csv", show_col_types = FALSE)
reviewers_csv <- read_csv("data/APD_reviewers.csv", show_col_types = FALSE)
references_csv <- read_csv("data/APD_references.csv", show_col_types = FALSE)
units_csv <- read_csv("data/APD_units.csv", show_col_types = FALSE)
hierarchy_csv <- read_csv("data/APD_trait_hierarchy.csv", show_col_types = FALSE) %>%
  mutate(Entity = paste0("https://w3id.org/APD/traits/", identifier))
categorical_values_csv <- read_csv("data/APD_categorical_values_input.csv", show_col_types = FALSE) %>%
  mutate(Entity = paste0("https://w3id.org/APD/traits/", identifier))
APD_resource_csv <- read_csv("data/APD_resource.csv", show_col_types = FALSE)

triples_df_tmp <- 
  convert_to_triples(annotation_properties_csv, traits_csv, glossary_csv, published_classes_csv, reviewers_csv, references_csv, units_csv, hierarchy_csv, categorical_values_csv, APD_resource_csv)

triples_df <- triples_df_tmp$triples_df
triples_with_labels <- triples_df_tmp$triples_with_labels
```

Export various formats

```{r}
triples_with_labels %>%
  dplyr::select(Subject, property, Predicate, value, Object, graph, Subject_stripped) %>%
  write_csv("APD_triples.csv")

triples_df %>%
  write_delim("APD.nq", col_names=FALSE, escape="none", quote="none")

triples_df %>%
  select(-graph) %>%
  write_delim("APD.nt", col_names = FALSE, escape = "none", quote = "none")

# prove this parses correctly
true_triples <- read_nquads("APD.nq")

# serialize to any format
rdflib::rdf_serialize(true_triples, "APD.ttl",
                      namespace = c(APD = "https://w3id.org/APD/traits/",
                                    APD_glossary = "https://w3id.org/APD/glossary/",
                                    dc = "http://purl.org/dc/elements/1.1/",
                                    skos = "http://www.w3.org/2004/02/skos/core#",
                                    dwc = "http://rs.tdwg.org/dwc/terms/attributes/",
                                    dcam = "http://purl.org/dc/dcam/",
                                    dcterms = "http://purl.org/dc/terms/",
                                    ets = "http://terminologies.gfbio.org/terms/ETS/",
                                    obo = "http://purl.obolibrary.org/obo/",
                                    oboecore = "http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#",
                                    ont = "https://w3id.org/iadopt/ont/",
                                    owl = "http://www.w3.org/2002/07/owl#",
                                    rdfs = "http://www.w3.org/2000/01/rdf-schema#",
                                    uom = "https://w3id.org/uom/",
                                    datacite = "http://purl.org/datacite/v4.4/",
                                    xsd = "http://www.w3.org/2001/XMLSchema#",
                                    Cerrado = "http://cerrado.linkeddata.es/ecology/",
                                    CorVeg  = "http://linked.data.gov.au/def/corveg-cv/",
                                    CO = "https://cropontology.org/rdf/",
                                    DCM = "http://dicom.nema.org/resources/ontology/DCM/",
                                    EDAM = "http://edamontology.org/",
                                    EFO = "http://www.ebi.ac.uk/efo/",
                                    EnvThes = "http://vocabs.lter-europe.net/EnvThes/",
                                    hupson = "http://scai.fraunhofer.de/HuPSON#",
                                    IOBC = "http://purl.jp/bio/4/id/",
                                    MESH = "http://purl.bioontology.org/ontology/MESH/",
                                    odo = "http://purl.dataone.org/odo/",
                                    ORCID = "https://orcid.org/",
                                    SIO = "http://semanticscience.org/resource/",
                                    SWEET_phenSolid = "http://sweetontology.net/phenSolid/",
                                    SWEET_phenSystem = "http://sweetontology.net/phenSystem/",
                                    SWEET_procWave = "http://sweetontology.net/procWave/",
                                    SWEET_prop = "http://sweetontology.net/prop/",
                                    SWEET_propConductivity = "http://sweetontology.net/propConductivity",
                                    SWEET_propPressure = "http://sweetontology.net/propPressure/",
                                    SWEET_propTime = "http://sweetontology.net/propTime/",
                                    SWEET_realmSoil = "http://sweetontology.net/realmSoil/",
                                    SWEET_reprSciComponent = "http://sweetontology.net/reprSciComponent/",
                                    SWEET_reprTimeDay = "http://sweetontology.net/reprTimeDay/")
                                    )
rdflib::rdf_serialize(true_triples, "APD.json", format="jsonld")
```


# Smoke-tests / example sparql queries
```{r}
# how many unique predicates?
sparql <-
'SELECT DISTINCT ?p
 WHERE { ?s ?p ?c . }
'
rdf_query(true_triples, sparql)

# how many unique reviewers are in the data?
sparql <-
'SELECT DISTINCT ?orcid ?prefLabel
 WHERE { ?s <http://purl.org/datacite/v4.4/IsReviewedBy> ?orcid .
         ?orcid <http://www.w3.org/2004/02/skos/core#prefLabel> ?prefLabel
       }
'

unescape_unicode <- function(x) {
  stringi::stri_unescape_unicode(gsub("<U\\+(....)>", "\\\\u\\1", x))
}

rdf_query(true_triples, sparql) %>%
  mutate(prefLabel = unescape_unicode(prefLabel))


# how many unique references are in the data?
sparql <-
  'SELECT DISTINCT ?id
 WHERE { ?s <http://purl.org/dc/terms/references> ?id .
       }
'
rdf_query(true_triples, sparql)


sparql <-
  'SELECT DISTINCT ?s
 WHERE { ?s <http://www.w3.org/2004/02/skos/core#prefLabel> "plant trait" .
       }
'
rdf_query(true_triples, sparql)
```

# Create a version of ouput for human access

```{r}
source("R/table.R")
source("R/helpers.R")
source("R/create_APD_trait_table.R")

base_url_traits <- "https://w3id.org/APD/traits/"
base_url_glossary <- "https://w3id.org/APD/glossary/"

# todo - these files already read in above, update to reuse objects()
APD_traits_input <- read_csv("data/APD_traits_input.csv", show_col_types = FALSE)
APD_reviewers <- read_csv("data/APD_reviewers.csv", show_col_types = FALSE)
APD_references <- read_csv("data/APD_references.csv", show_col_types = FALSE)
APD_hierarchy <- read_csv("data/APD_trait_hierarchy.csv", show_col_types = FALSE)
published_classes <- read_csv("data/published_classes.csv", show_col_types = FALSE)
categorical <- read_csv("data/APD_categorical_values_input.csv", show_col_types = FALSE)

# information direct from traits table
core_traits <- 
  APD_traits_input %>%
  dplyr::select(dplyr::all_of(c(
    "trait", "label", "description_encoded", "description", "comments", "type", "min", "max", "units", "constraints",
    "created", "modified", "reviewed", "deprecated_trait_name", "identifier", "inScheme"
  ))) %>%
  dplyr::rename(dplyr::all_of(c(
    "allowed_values_min" = "min",
    "allowed_values_max" = "max",
    "trait_type" = "type"
  ))) %>%
  # add label to trait type
  dplyr::mutate(
    Entity = paste0(inScheme, "/traits/", identifier),
    trait_type = paste0(published_classes$label[match(trait_type, published_classes$identifier)], " [", trait_type, "]")
  )

# collapse traits table to include a single column for each property

# collapse reviews, add ORCIDs
reviewers <- 
  APD_traits_input %>%
  dplyr::select(trait, reviewers) %>%
  dplyr::mutate(reviewers = stringr::str_split(reviewers, "; ")) %>%
  tidyr::unnest_wider(col = reviewers, names_sep = "_") 

reviewers <- reviewers %>%
  tidyr::pivot_longer(cols = 2:ncol(reviewers)) %>%
  dplyr::filter(!is.na(value)) %>%
  dplyr::mutate(
    value = ifelse(!is.na(value), paste0(value, " [", APD_reviewers$Entity[match(value, APD_reviewers$label)], "]"), NA)
  ) %>%
  group_by(trait) %>%
  dplyr::mutate(reviewers = paste(value, collapse = "; ")) %>%
  ungroup() %>%
  dplyr::select(-dplyr::all_of(c("name", "value"))) %>%
  dplyr::distinct()

# collapse references, add DOIs
references <- 
  APD_traits_input %>%
  dplyr::select(trait, references) %>%
  dplyr::mutate(references = stringr::str_split(references, "; ")) %>%
  tidyr::unnest_wider(col = references, names_sep = "_") 
  
references <- references %>%  
  tidyr::pivot_longer(cols = 2:ncol(references)) %>%
  dplyr::filter(!is.na(value)) %>%
  dplyr::mutate(
    value = ifelse(!is.na(value), paste0(value, " [", APD_references$Entity[match(value, APD_references$label)], "]"), NA)
  ) %>%
  dplyr::group_by(trait) %>%
  dplyr::mutate(references = paste(value, collapse = "; ")) %>%
  dplyr::ungroup() %>%
  dplyr::select(-dplyr::all_of(c("name", "value"))) %>%
  dplyr::distinct()

# collapse broader groupings
hierarchy <- 
  APD_traits_input %>%
  dplyr::select(trait, category) %>%
  dplyr::mutate(category = stringr::str_split(category, "; ")) %>%
  tidyr::unnest_wider(col = category, names_sep = "_") 

hierarchy <- hierarchy %>%
  tidyr::pivot_longer(cols = 2:ncol(hierarchy)) %>%
  dplyr::filter(!is.na(value)) %>%
  dplyr::mutate(
    value = stringr::str_replace(value, "APD\\:", ""),
    value = ifelse(!is.na(value), paste0(APD_hierarchy$label[match(value, APD_hierarchy$identifier)], " [", value, "]"), NA)
  ) %>%
  dplyr::group_by(trait) %>%
  dplyr::mutate(trait_groupings = paste(value, collapse = "; ")) %>%
  dplyr::ungroup() %>%
  dplyr::select(-dplyr::all_of(c("name", "value"))) %>%
  dplyr::distinct()

# collapse structures measured; replace class identifiers with labels
structure_measured <- 
  APD_traits_input %>%
  dplyr::select(trait, structure) %>%
  dplyr::mutate(structure = stringr::str_split(structure, "; ")) %>%
  tidyr::unnest_wider(col = structure, names_sep = "_") 
  
structure_measured <- structure_measured %>%
  tidyr::pivot_longer(cols = 2:ncol(structure_measured)) %>%
  dplyr::filter(!is.na(value)) %>%
  dplyr::mutate(
    value = ifelse(!is.na(value), paste0(published_classes$label[match(value, published_classes$identifier)], " [", value, "]"), NA)
  ) %>%
  dplyr::group_by(trait) %>%
  dplyr::mutate(structure_measured = paste(value, collapse = "; ")) %>%
  dplyr::ungroup() %>%
  dplyr::select(-dplyr::all_of(c("name", "value"))) %>%
  dplyr::distinct()

# collapse characteristics measured; replace class identifiers with labels
characteristic_measured <- 
  APD_traits_input %>%
  dplyr::select(trait, measured_characteristic) %>%
  dplyr::mutate(measured_characteristic = stringr::str_split(measured_characteristic, "; ")) %>%
  tidyr::unnest_wider(col = measured_characteristic, names_sep = "_") 
  
characteristic_measured <- characteristic_measured %>%  
  tidyr::pivot_longer(cols = 2:ncol(characteristic_measured)) %>%
  dplyr::filter(!is.na(value)) %>%
  dplyr::mutate(
    value = ifelse(!is.na(value), paste0(published_classes$label[match(value, published_classes$identifier)], " [", value, "]"), NA)
  ) %>%
  dplyr::group_by(trait) %>%
  dplyr::mutate(characteristic_measured = paste(value, collapse = "; ")) %>%
  dplyr::ungroup() %>%
  dplyr::select(-dplyr::all_of(c("name", "value"))) %>%
  dplyr::distinct()

# collapse keywords; replace class identifiers with labels
keywords <- 
  APD_traits_input %>%
  dplyr::select(trait, keywords) %>%
  dplyr::mutate(keywords = stringr::str_split(keywords, "; ")) %>%
  tidyr::unnest_wider(col = keywords, names_sep = "_") 
  
keywords <- keywords %>%  
  tidyr::pivot_longer(cols = 2:ncol(keywords)) %>%
  dplyr::filter(!is.na(value)) %>%
  dplyr::mutate(
    value = ifelse(!is.na(value), paste0(published_classes$label[match(value, published_classes$identifier)], " [", value, "]"), NA)
  ) %>%
  dplyr::group_by(trait) %>%
  dplyr::mutate(keywords = paste(value, collapse = "; ")) %>%
  dplyr::ungroup() %>%
  dplyr::select(-dplyr::all_of(c("name", "value"))) %>%
  dplyr::distinct()

# exact matches to published vocabularies
exact <- 
  APD_traits_input %>%
  dplyr::select(trait, exact_match) %>%
  dplyr::mutate(exact_match = stringr::str_split(exact_match, "; ")) %>%
  tidyr::unnest_wider(col = exact_match, names_sep = "_") 
  
exact <- exact %>% 
  tidyr::pivot_longer(cols = 2:ncol(exact)) %>%
  dplyr::filter(!is.na(value)) %>%
  dplyr::mutate(
    exact_match = paste0(
      published_classes$label[match(value, published_classes$identifier)], " [",
      published_classes$Entity[match(value, published_classes$identifier)], "]"
    )
  ) %>%
  dplyr::group_by(trait) %>%
  dplyr::mutate(exact_match = paste(exact_match, collapse = "; ")) %>%
  dplyr::ungroup() %>%
  dplyr::select(-dplyr::all_of(c("name", "value"))) %>%
  dplyr::distinct()

# close matches to published vocabularies
close <- 
  APD_traits_input %>%
  dplyr::select(trait, close_match) %>%
  dplyr::mutate(close_match = stringr::str_split(close_match, "; ")) %>%
  tidyr::unnest_wider(col = close_match, names_sep = "_") 
  
close <- close %>% 
  tidyr::pivot_longer(cols = 2:ncol(close)) %>%
  dplyr::filter(!is.na(value)) %>%
  dplyr::mutate(
    close_match = paste0(
      published_classes$label[match(value, published_classes$identifier)], " [",
      published_classes$Entity[match(value, published_classes$identifier)], "]"
    )
  ) %>%
  dplyr::group_by(trait) %>%
  dplyr::mutate(close_match = paste(close_match, collapse = "; ")) %>%
  dplyr::ungroup() %>%
  dplyr::select(-dplyr::all_of(c("name", "value"))) %>%
  dplyr::distinct()


# close matches to published vocabularies
related <- 
  APD_traits_input %>%
  dplyr::select(trait, related_match) %>%
  dplyr::mutate(related_match = stringr::str_split(related_match, "; ")) %>%
  tidyr::unnest_wider(col = related_match, names_sep = "_") 
  
related <- related %>% 
  tidyr::pivot_longer(cols = 2:ncol(related)) %>%
  dplyr::filter(!is.na(value)) %>%
  dplyr::mutate(
    related_match = paste0(
      published_classes$label[match(value, published_classes$identifier)], " [",
      published_classes$Entity[match(value, published_classes$identifier)], "]"
    )
  ) %>%
  dplyr::group_by(trait) %>%
  dplyr::mutate(related_match = paste(related_match, collapse = "; ")) %>%
  dplyr::ungroup() %>%
  dplyr::select(-dplyr::all_of(c("name", "value"))) %>%
  dplyr::distinct()


examples <- 
  APD_traits_input %>%
  dplyr::select(trait, dplyr::contains("_exact"), dplyr::contains("_related"), dplyr::contains("_close"))

examples <- examples %>%
  mutate(across(c(2:ncol(examples)), ~ stringr::str_split(.x, "; "))) %>%
  tidyr::unnest_wider(col = c(2:ncol(examples)), names_sep = "_") %>%
  mutate(
    across(dplyr::contains("_exact"), \(x) ifelse(!is.na(x), paste0("exact match: ", x), NA)),
    across(dplyr::contains("_close"), \(x) ifelse(!is.na(x), paste0("close match: ", x), NA)),
    across(dplyr::contains("_related"), \(x) ifelse(!is.na(x), paste0("related match: ", x), NA))
  )

examples <- examples %>%
  tidyr::pivot_longer(cols = 2:ncol(examples)) %>%
  filter(!is.na(value)) %>%
  dplyr::group_by(trait) %>%
  dplyr::mutate(examples = paste(value, collapse = "; ")) %>%
  dplyr::ungroup() %>%
  dplyr::select(-dplyr::all_of(c("name", "value"))) %>%
  dplyr::distinct()

# categorical trait values
categorical %>%
  mutate(
    description = stringr::str_split(description, "[:space:]\\(Synonym\\, |[:space:]\\(Synonyms\\, ")
  ) %>%
  tidyr::unnest_wider(description, names_sep = "_") %>%
  dplyr::mutate(description_2 = stringr::str_replace(description_2, "\\)$", "")) %>%
  dplyr::rename(
    allowed_values_levels = label,
    trait = trait_name,
    categorical_trait_description = description_1,
    categorical_trait_synonyms = description_2,
    categorical_trait_identifier = identifier
  ) %>%
  dplyr::select(allowed_values_levels, trait, categorical_trait_description, categorical_trait_synonyms, categorical_trait_identifier) %>%
  write_csv("APD_categorical_values.csv")

# join together pieces
core_traits %>%
  dplyr::left_join(hierarchy, by = c("trait")) %>%
  dplyr::left_join(structure_measured, by = c("trait")) %>%
  dplyr::left_join(characteristic_measured, by = c("trait")) %>%
  dplyr::left_join(keywords, by = c("trait")) %>%
  dplyr::left_join(references, by = c("trait")) %>%
  dplyr::left_join(reviewers, by = c("trait")) %>%
  dplyr::left_join(exact, by = c("trait")) %>%
  dplyr::left_join(close, by = c("trait")) %>%
  dplyr::left_join(related, by = c("trait")) %>%
  dplyr::left_join(examples, by = c("trait")) %>%
  # sort columns
  dplyr::select(dplyr::all_of(c(
    "Entity", "trait", "label", "description", "comments", "trait_type", "allowed_values_min", "allowed_values_max", "units",
    "constraints", "trait_groupings", "structure_measured", "characteristic_measured", "keywords",
    "references", "reviewers", "created", "modified",
    "exact_match", "close_match", "related_match", "examples",
    "description_encoded", "deprecated_trait_name", "identifier", "inScheme"
  ))) %>%
  write_csv("APD_traits.csv")
```

# STOP HERE FOR INDIVIDUAL BRANCHES
#   REMAINING CODE JUST FOR RELEASES

# Build the website - only do this for releases
```{r}
quarto::quarto_render()
```

# Website - only do this for releases

Put a copy of latest in release folder in both `release` and `docs/release` folders

```{r}
version <- rmarkdown::yaml_front_matter("index.qmd")$params$version

files <- c("APD.json", "APD.nq", "APD.nt", "APD.ttl", "APD_traits.csv", "APD_categorical_values.csv", "index.html")

for(p in c("release", "docs/release")) {
  to_path <- file.path(p, version)
  dir.create(to_path, FALSE, TRUE)
  purrr::walk(files, ~ file.copy(file.path("docs", .x), file.path(to_path, .x), overwrite = TRUE))
}
```