en.py

"""Utilities to derive an EDTF string from an (English) natural language string."""

import re
from datetime import datetime

from dateutil.parser import ParserError, parse

from edtf import appsettings

# two dates where every digit of an ISO date representation is different,
# and one is in the past and one is in the future.
# This is used in the dateutil parse to detect which elements weren't parsed.
DEFAULT_DATE_1 = datetime(1234, 1, 1, 0, 0)
DEFAULT_DATE_2 = datetime(5678, 10, 10, 0, 0)

SHORT_YEAR_RE = r"(-?)([\dX])([\dX])([\dX])([\dX])"
LONG_YEAR_RE = r"Y(-?)([1-9]\d\d\d\d+)"
CENTURY_RE = r"(\d{1,2})(c\.?|(st|nd|rd|th) century)\s?(ad|ce|bc|bce)?"
CE_RE = r"(\d{1,4}) (ad|ce|bc|bce)"

# Set of RE rules that will cause us to abort text processing, since we know
# the results will be wrong.
REJECT_RULES = (
    r".*dynasty.*",  # Don't parse '23rd Dynasty' to 'uuuu-uu-23'
)


def text_to_edtf(text):
    """
    Generate EDTF string equivalent of a given natural language date string.
    """
    if not text:
        return None

    t = text.lower()

    # try parsing the whole thing
    result = text_to_edtf_date(t)

    if not result:
        # split by list delims and move fwd with the first thing that returns a non-empty string.
        # TODO: assemble multiple dates into a {} or [] structure.
        for split in [",", ";", "or"]:
            for list_item in t.split(split):
                # try parsing as an interval - split by '-'
                toks = list_item.split("-")
                if len(toks) == 2:
                    d1 = toks[0].strip()
                    d2 = toks[1].strip()

                    # match looks from the beginning of the string, search
                    # looks anywhere.

                    if re.match(r"\d\D\b", d2):  # 1-digit year partial e.g. 1868-9
                        if re.search(
                            r"\b\d\d\d\d$", d1
                        ):  # TODO: evaluate it and see if it's a year
                            d2 = d1[-4:-1] + d2
                    elif re.match(r"\d\d\b", d2):  # 2-digit year partial e.g. 1809-10
                        if re.search(r"\b\d\d\d\d$", d1):
                            d2 = d1[-4:-2] + d2
                    else:
                        century_range_match = re.search(
                            r"\b(\d\d)(th|st|nd|rd|)-(\d\d)(th|st|nd|rd) [cC]",
                            f"{d1}-{d2}",
                        )
                        if century_range_match:
                            g = century_range_match.groups()
                            d1 = f"{g[0]}C"
                            d2 = f"{g[2]}C"

                    r1 = text_to_edtf_date(d1)
                    r2 = text_to_edtf_date(d2)

                    if r1 and r2:
                        result = r1 + "/" + r2
                        return result

                # is it an either/or year "1838/1862" - that has a different
                # representation in EDTF. If it's 'both', then we use {}. If
                # it's 'or' then we use []. Assuming the latter for now.
                # This whole section could be more friendly.

                else:
                    int_match = re.search(r"(\d\d\d\d)\/(\d\d\d\d)", list_item)
                    if int_match:
                        return f"[{int_match.group(1)}, {int_match.group(2)}]"

                result = text_to_edtf_date(list_item)
                if result:
                    break
            if result:
                break

    is_before = re.findall(r"\bbefore\b", t)
    is_before = is_before or re.findall(r"\bearlier\b", t)

    is_after = re.findall(r"\bafter\b", t)
    is_after = is_after or re.findall(r"\bsince\b", t)
    is_after = is_after or re.findall(r"\blater\b", t)

    if is_before:
        result = f"/{result}"  # unknown is replaced with null for intervals
    elif is_after:
        result = f"{result}/"  # unknown is replaced with null for intervals
    return result


def text_to_edtf_date(text):
    """
    Return EDTF string equivalent of a given natural language date string.

    The approach here is to parse the text twice, with different default
    dates. Then compare the results to see what differs - the parts that
    differ are undefined.
    """
    if not text:
        return

    t = text.lower()
    result = ""

    for reject_re in REJECT_RULES:
        if re.match(reject_re, t):
            return

    # matches on '1800s'. Needs to happen before is_decade.
    could_be_century = re.findall(r"(\d{2}00)s", t)
    # matches on '1800s' and '1910s'. Removes the 's'.
    # Needs to happen before is_uncertain because e.g. "1860s?"
    t, is_decade = re.subn(r"(\d{3}0)s", r"\1", t)

    # detect approximation signifiers
    # a few 'circa' abbreviations just before the year
    is_approximate = re.findall(r"\b(ca?\.?) ?\d{4}", t)
    # the word 'circa' anywhere
    is_approximate = is_approximate or re.findall(r"\bcirca\b", t)
    # the word 'approx'/'around'/'about' anywhere
    is_approximate = is_approximate or re.findall(r"\b(approx|around|about)", t)
    # a ~ before a year-ish number
    is_approximate = is_approximate or re.findall(r"\b~\d{4}", t)
    # a ~ at the beginning
    is_approximate = is_approximate or re.findall(r"^~", t)

    # detect uncertainty signifiers
    t, is_uncertain = re.subn(r"(\d{4})\?", r"\1", t)
    # the words uncertain/maybe/guess anywhere
    is_uncertain = is_uncertain or re.findall(r"\b(uncertain|possibly|maybe|guess)", t)

    # detect century forms
    is_century = re.findall(CENTURY_RE, t)

    # detect CE/BCE year form
    is_ce = re.findall(CE_RE, t)
    if is_century:
        result = "%02dXX" % (int(is_century[0][0]) - 1,)
        is_approximate = is_approximate or re.findall(r"\b(ca?\.?) ?" + CENTURY_RE, t)
        is_uncertain = is_uncertain or re.findall(CENTURY_RE + r"\?", t)

        try:
            is_bc = is_century[0][-1] in ("bc", "bce")
            if is_bc:
                result = f"-{result}"
        except IndexError:
            pass

    elif is_ce:
        result = "%04d" % (int(is_ce[0][0]))
        is_approximate = is_approximate or re.findall(r"\b(ca?\.?) ?" + CE_RE, t)
        is_uncertain = is_uncertain or re.findall(CE_RE + r"\?", t)

        try:
            is_bc = is_ce[0][-1] in ("bc", "bce")
            if is_bc:
                result = f"-{result}"
        except IndexError:
            pass

    else:
        # try dateutil.parse

        try:
            # parse twice, using different defaults to see what was
            # parsed and what was guessed.
            dt1 = parse(
                t,
                dayfirst=appsettings.DAY_FIRST,
                yearfirst=False,
                fuzzy=True,  # force a match, even if it's default date
                default=DEFAULT_DATE_1,
            )

            dt2 = parse(
                t,
                dayfirst=appsettings.DAY_FIRST,
                yearfirst=False,
                fuzzy=True,  # force a match, even if it's default date
                default=DEFAULT_DATE_2,
            )

        except ParserError:
            return
        except Exception:
            return

        if dt1.date() == DEFAULT_DATE_1.date() and dt2.date() == DEFAULT_DATE_2.date():
            # couldn't parse anything - defaults are untouched.
            return

        date1 = dt1.isoformat()[:10]
        date2 = dt2.isoformat()[:10]

        # guess precision of 'unspecified' characters to use
        mentions_year = re.findall(r"\byear\b.+(in|during)\b", t)
        mentions_month = re.findall(r"\bmonth\b.+(in|during)\b", t)
        mentions_day = re.findall(r"\bday\b.+(in|during)\b", t)

        for i in range(len(date1)):
            # if the given year could be a century (e.g. '1800s') then use
            # approximate/uncertain markers to decide whether we treat it as
            # a century or a decade.
            if i == 2 and could_be_century and not (is_approximate or is_uncertain):
                result += "X"
            elif i == 3 and is_decade > 0:
                if mentions_year:
                    result += "X"  # previously year precision - now just X
                else:
                    result += "X"  # previously decade precision - now just X
            elif date1[i] == date2[i]:
                # since both attempts at parsing produced the same result
                # it must be parsed value, not a default
                result += date1[i]
            else:
                # different values were produced, meaning that it's likely
                # a default. Use 'X'
                result += "X"

        # strip off unknown chars from end of string - except the first 4

        for i in reversed(range(len(result))):
            if result[i] not in ("X", "-"):
                smallest_length = 4

                if mentions_month:
                    smallest_length = 7
                if mentions_day:
                    smallest_length = 10

                limit = max(smallest_length, i + 1)
                result = result[:limit]
                break

        # check for seasons
        if "spring" in t:
            result = result[:4] + "-21" + result[7:]
        elif "summer" in t:
            result = result[:4] + "-22" + result[7:]
        elif "autumn" in t or "fall" in t:
            result = result[:4] + "-23" + result[7:]
        elif "winter" in t:
            result = result[:4] + "-24" + result[7:]

            # end dateutil post-parsing

    if is_uncertain and is_approximate:
        result += "%"
    else:
        if is_uncertain:
            result += "?"
        if is_approximate:
            result += "~"

    # weed out bad parses
    if result.startswith("XX-XX"):
        return None

    return result