language_identification.schema.json

{
  "$schema": "http://json-schema.org/draft-06/schema#",
  "$id": "https://impresso.github.io/impresso-schemas/json/language_identification/language_identification.schema.json",
  "title": "Impresso Language Identification Information",
  "description": "A representation for information relevant to impresso's content item language recognition. This defines the output of the following impresso NLP component. https://github.com/impresso/impresso-language-identification ",
  "type": "object",
  "properties": {
    "id": {
      "type": "string",
      "description": "The unique identifier for a content item, cf. https://github.com/impresso/impresso-schemas/blob/master/json/newspaper/contentitem.schema.json"
    },
    "orig_lg": {
      "type": [
        "string",
        "null"
      ],
      "oneOf": [
        {
          "title": "null",
          "description": "No information on language from original metadata is available.",
          "type": "null"
        },
        {
          "$ref": "#/definitions/langISO639Type"
        }
      ],
      "description": "Original language of the content item as provided by the metadata, cf. https://github.com/impresso/impresso-schemas/blob/master/json/newspaper/contentitem.schema.json"
    },
    "lg": {
      "type": [
        "string",
        "null"
      ],
      "oneOf": [
        {
          "title": "null",
          "type": "null"
        },
        {
          "$ref": "#/definitions/langISO639Type"
        }
      ],
      "description": "Computed language of the content item, cf. https://github.com/impresso/impresso-schemas/blob/master/json/newspaper/contentitem.schema.json"
    },
    "lg_decision": {
      "enum": [
        "all",
        "all-but-impresso_ft",
        "voting",
        "dominant-by-len",
        "dominant-by-lowvote"
      ],
      "type": "string",
      "description": "An identifier for the decision strategy applied to the content item: 'all' = all LID systems/info agree; 'all-but-impresso_ft' = all LID except impresso_ft agree on a language other than de/fr; 'dominant-by-len' = the most frequent language of the ensemble decisions is selected because there are too few characters; 'dominant-by-lowvote' = the most frequent language of the ensemble decisions is selected because there are too few votes; 'voting' = the language with the highest vote count is selected "
    },
    "tp": {
      "type": "string",
      "description": "Type of the content item, cf. https://github.com/impresso/impresso-schemas/blob/master/json/newspaper/contentitem.schema.json"
    },
    "len": {
      "type": "integer",
      "description": "Number of characters of content item",
      "minimum": 0
    },
    "impresso_language_identifier_version": {
      "$ref": "#/definitions/versionType",
      "description": "Version information of used impresso language identifier script and the file creation timestamp"
    },
    "language_identifier_version": {
      "$ref": "#/definitions/versionType",
      "description": "Version information of used impresso language identifier script and the timestamp of the file used for the decision"
    },
    "alphabetical_ratio": {
      "type": [
        "number",
        "null"
      ],
      "description": "Ratio of alphabetical characters (Unicode letter class matched by \\w) w.r.t. text length of content item",
      "minimum": 0,
      "maximum": 1
    },
    "langdetect": {
      "type": "array",
      "items": {
        "$ref": "#/definitions/lidType"
      },
      "description": "List of predicted languages using Langdetect, sorted by probability"
    },
    "langid": {
      "type": "array",
      "items": {
        "$ref": "#/definitions/lidType"
      },
      "description": "List of predicted languages using Langid, sorted by probability"
    },
    "impresso_ft": {
      "type": "array",
      "items": {
        "$ref": "#/definitions/lidType"
      },
      "description": "List of predicted languages using a FastText model trained on Impresso articles, sorted by probability"
    },
    "wp_ft": {
      "type": "array",
      "items": {
        "$ref": "#/definitions/lidType"
      },
      "description": "List of predicted languages using a FastText model trained on Wikipedia, sorted by probability"
    },
    "votes": {
      "type": "array",
      "items": [
        {
          "type": "object",
          "properties": {
            "lang": {
              "$ref": "#/definitions/langISO639Type"
            },
            "vote": {
              "type": "number"
            }
          },
          "required": [
            "lang",
            "vote"
          ]
        }
      ]
    }
  },
  "required": [
    "tp",
    "id",
    "len",
    "lg",
    "orig_lg"
  ],
  "definitions": {
    "lidType": {
      "title": "lidType",
      "description": "Predicted language and its probability",
      "type": "object",
      "properties": {
        "lang": {
          "$ref": "#/definitions/langISO639Type"
        },
        "prob": {
          "$ref": "#/definitions/probType"
        }
      },
      "required": [
        "lang",
        "prob"
      ]
    },
    "langISO639Type": {
      "type": "string",
      "title": "langISO639Type",
      "description": "Two- or three-letter ISO language abbreviation, cf. https://en.wikipedia.org/wiki/List_of_ISO_639-2_codes or https://en.wikipedia.org/wiki/List_of_ISO_639-3_codes",
      "pattern": "^[a-z]{2,3}$"
    },
    "probType": {
      "type": "number",
      "title": "probType",
      "description": "Probability of the prediction",
      "minimum": 0,
      "maximum": 1
    },
    "versionType": {
      "title": "versionType",
      "description": "Administrative information on tool version and data creation time.",
      "type": "object",
      "properties": {
        "versionType": {
          "description": "Version of tool. Either date or output of command 'git describe'",
          "type": "string"
        },
        "ts": {
          "description": "timestamp of creation of the JSON file (e.g. '2018-09-18T08:00:08+00:00')",
          "type": "string",
          "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(Z|\\+00:00)$"
        }
      },
      "required": [
        "ts",
        "version"
      ]
    }
  }
}