Skip to content

Commit

Permalink
3.9+ updates for main lib
Browse files Browse the repository at this point in the history
  • Loading branch information
devdupont committed May 26, 2024
1 parent 9a534fd commit d44d949
Show file tree
Hide file tree
Showing 51 changed files with 953 additions and 1,106 deletions.
4 changes: 2 additions & 2 deletions avwx/current/airsigmet.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
from avwx.flight_path import to_coordinates
from avwx.load_utils import LazyLoad
from avwx.parsing import core
from avwx.service.bulk import NOAA_Bulk, NOAA_Intl, Service
from avwx.service.bulk import NoaaBulk, NoaaIntl, Service
from avwx.static.airsigmet import BULLETIN_TYPES, INTENSITY, WEATHER_TYPES
from avwx.static.core import CARDINAL_DEGREES, CARDINALS
from avwx.structs import (
Expand Down Expand Up @@ -152,7 +152,7 @@ class AirSigManager:
reports: list[AirSigmet] | None = None

def __init__(self): # type: ignore
self._services = [NOAA_Bulk("airsigmet"), NOAA_Intl("airsigmet")]
self._services = [NoaaBulk("airsigmet"), NoaaIntl("airsigmet")]
self._raw, self.raw = [], []

async def _update(self, index: int, timeout: int) -> list[tuple[str, str | None]]:
Expand Down
6 changes: 3 additions & 3 deletions avwx/current/metar.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
from avwx.parsing import core, remarks, speech, summary
from avwx.parsing.sanitization.metar import clean_metar_list, clean_metar_string
from avwx.parsing.translate.metar import translate_metar
from avwx.service import NOAA
from avwx.service import Noaa
from avwx.static.core import FLIGHT_RULES
from avwx.static.metar import METAR_RMK
from avwx.station import uses_na_format, valid_station
Expand Down Expand Up @@ -83,7 +83,7 @@ class Metar(Report):

async def _pull_from_default(self) -> None:
"""Check for a more recent report from NOAA."""
service = NOAA(self.__class__.__name__.lower())
service = Noaa(self.__class__.__name__.lower())
if self.code is None:
return
report = await service.async_fetch(self.code)
Expand All @@ -98,7 +98,7 @@ async def _pull_from_default(self) -> None:
@property
def _should_check_default(self) -> bool:
"""Return True if pulled from regional source and potentially out of date."""
if isinstance(self.service, NOAA) or self.source is None:
if isinstance(self.service, Noaa) or self.source is None:
return False

if self.data is None or self.data.time is None or self.data.time.dt is None:
Expand Down
4 changes: 2 additions & 2 deletions avwx/current/notam.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
from avwx import exceptions
from avwx.current.base import Reports
from avwx.parsing import core
from avwx.service import FAA_NOTAM
from avwx.service import FaaNotam
from avwx.static.core import SPECIAL_NUMBERS
from avwx.static.notam import (
CODES,
Expand Down Expand Up @@ -139,7 +139,7 @@ class Notams(Reports):

def __init__(self, code: str | None = None, coord: Coord | None = None):
super().__init__(code, coord)
self.service = FAA_NOTAM("notam")
self.service = FaaNotam("notam")

async def _post_update(self) -> None:
self._post_parse()
Expand Down
4 changes: 2 additions & 2 deletions avwx/current/pirep.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
from avwx.current.base import Reports, get_wx_codes
from avwx.parsing import core
from avwx.parsing.sanitization.pirep import clean_pirep_string
from avwx.service.scrape import NOAA_ScrapeList
from avwx.service.scrape import NoaaScrapeList
from avwx.static.core import CARDINALS, CLOUD_LIST
from avwx.structs import (
Aircraft,
Expand Down Expand Up @@ -69,7 +69,7 @@ class Pireps(Reports):

def __init__(self, code: str | None = None, coord: Coord | None = None):
super().__init__(code, coord)
self.service = NOAA_ScrapeList("pirep")
self.service = NoaaScrapeList("pirep")

@staticmethod
def _report_filter(reports: list[str]) -> list[str]:
Expand Down
4 changes: 1 addition & 3 deletions avwx/data/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,7 @@

def update_all() -> bool:
"""Update all local data. Requires a reimport to guarentee update"""
return not any(
func() for func in (update_aircraft, update_navaids, update_stations)
)
return not any(func() for func in (update_aircraft, update_navaids, update_stations))


__all__ = ["update_all", "update_aircraft", "update_navaids", "update_stations"]
Expand Down
7 changes: 2 additions & 5 deletions avwx/data/build_aircraft.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
"""
Builds the aircraft code dict
"""
"""Builds the aircraft code dict."""

# stdlib
import json
Expand All @@ -10,14 +8,13 @@
# library
import httpx


URL = "https://en.wikipedia.org/wiki/List_of_ICAO_aircraft_type_designators"
OUTPUT_PATH = Path(__file__).parent / "files" / "aircraft.json"
TAG_PATTERN = re.compile(r"<[^>]*>")


def main() -> int:
"""Builds/updates aircraft.json codes"""
"""Build/update aircraft.json codes."""
resp = httpx.get(URL)
if resp.status_code != 200:
return 1
Expand Down
10 changes: 4 additions & 6 deletions avwx/data/build_navaids.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
"""
Build navaid coordinate map
"""
"""Build navaid coordinate map."""

import json
from pathlib import Path
from typing import Dict, Set, Tuple

import httpx

# redirect https://ourairports.com/data/navaids.csv
Expand All @@ -13,11 +11,11 @@


def main() -> None:
"""Builds the navaid coordinate map"""
"""Build the navaid coordinate map."""
text = httpx.get(URL).text
lines = text.strip().split("\n")
lines.pop(0)
data: Dict[str, Set[Tuple[float, float]]] = {}
data: dict[str, set[tuple[float, float]]] = {}
for line_str in lines:
line = line_str.split(",")
try:
Expand Down
67 changes: 35 additions & 32 deletions avwx/data/build_stations.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
"""
Builds the main station list
"""Builds the main station list.
Source file for airports.csv and runways.csv can be downloaded from
http://ourairports.com/data/
Expand All @@ -9,20 +8,24 @@
"""

# stdlib
from __future__ import annotations

import csv
import json
import logging
from contextlib import suppress
from datetime import date
from datetime import datetime, timezone
from pathlib import Path
from typing import Dict, Iterable, List, Optional, Tuple
from typing import TYPE_CHECKING

# library
import httpx

# module
from avwx.data.mappers import FILE_REPLACE, SURFACE_TYPES

if TYPE_CHECKING:
from collections.abc import Iterable

LOG = logging.getLogger("avwx.data.build_stations")

Expand All @@ -40,7 +43,7 @@ def load_stations(path: Path) -> Iterable[str]:

DATA_ROOT = "https://davidmegginson.github.io/ourairports-data/"
REPO_ROOT = "https://raw.githubusercontent.com/avwx-rest/avwx-engine/main/data/"
_SOURCE: Dict[str, str] = {}
_SOURCE: dict[str, str] = {}
_SOURCES = {
"airports": f"{DATA_ROOT}airports.csv",
"runways": f"{DATA_ROOT}runways.csv",
Expand All @@ -51,9 +54,9 @@ def load_stations(path: Path) -> Iterable[str]:


# Managed list of official ICAO idents
ICAO: List[str] = []
ICAO: list[str] = []
# Allow-listed AWOS stations not covered by ICAO-GPS codes
AWOS: List[str] = []
AWOS: list[str] = []


ACCEPTED_STATION_TYPES = [
Expand All @@ -68,34 +71,34 @@ def load_stations(path: Path) -> Iterable[str]:


def nullify(data: dict) -> dict:
"""Nullify empty strings in a dict"""
"""Nullify empty strings in a dict."""
for key, val in data.items():
if isinstance(val, str) and not val.strip():
data[key] = None
return data


def format_coord(coord: str) -> float:
"""Convert coord string to float"""
"""Convert coord string to float."""
neg = -1 if coord[-1] in ("S", "W") else 1
return neg * float(coord[:-1].strip().replace(" ", "."))


def load_codes() -> None:
"""Load ident lists"""
"""Load ident lists."""
# Global can't assign
for key, out in (("icaos", ICAO), ("awos", AWOS)):
for code in json.loads(_SOURCE[key]):
out.append(code)


def validate_icao(code: str) -> Optional[str]:
"""Validates a given station ident"""
def validate_icao(code: str) -> str | None:
"""Validate a given station ident."""
return None if len(code) != 4 and code not in AWOS else code.upper()


def get_icao(station: List[str]) -> Optional[str]:
"""Finds the ICAO by checking ident and GPS code"""
def get_icao(station: list[str]) -> str | None:
"""Find the ICAO by checking ident and GPS code."""
gps_code = validate_icao(station[12])
if gps_code and gps_code in ICAO:
return gps_code
Expand All @@ -104,15 +107,15 @@ def get_icao(station: List[str]) -> Optional[str]:


def clean_source_files() -> None:
"""Cleans the source data files before parsing"""
"""Clean the source data files before parsing."""
text = _SOURCE["airports"]
for find, replace in FILE_REPLACE.items():
text = text.replace(find, replace)
_SOURCE["airports"] = text


def format_station(code: str, station: List[str]) -> dict:
"""Converts source station list into info dict"""
def format_station(code: str, station: list[str]) -> dict:
"""Convert source station list into info dict."""
try:
elev_ft = float(station[6])
elev_m = round(elev_ft * 0.3048)
Expand Down Expand Up @@ -142,8 +145,8 @@ def format_station(code: str, station: List[str]) -> dict:
return nullify(ret)


def build_stations() -> Tuple[dict, dict]:
"""Builds the station dict from source file"""
def build_stations() -> tuple[dict, dict]:
"""Build the station dict from source file."""
stations, code_map = {}, {}
data = csv.reader(_SOURCE["airports"].splitlines())
next(data) # Skip header
Expand All @@ -156,18 +159,18 @@ def build_stations() -> Tuple[dict, dict]:


def add_missing_stations(stations: dict) -> dict:
"""Add non-airport stations from NOAA extract"""
stations.update(json.loads(_SOURCE["weather_stations"]))
"""Add non-airport stations from NOAA extract."""
stations |= json.loads(_SOURCE["weather_stations"])
return stations


def get_surface_type(surface: str) -> Optional[str]:
"""Returns the normalize surface type value"""
def get_surface_type(surface: str) -> str | None:
"""Return the normalize surface type value."""
return next((key for key, items in SURFACE_TYPES.items() if surface in items), None)


def add_runways(stations: dict, code_map: dict) -> dict:
"""Add runway information to station if availabale"""
"""Add runway information to station if availabale."""
data = csv.reader(_SOURCE["runways"].splitlines())
next(data) # Skip header
for runway in data:
Expand Down Expand Up @@ -200,31 +203,31 @@ def add_runways(stations: dict, code_map: dict) -> dict:


def add_reporting(stations: dict) -> dict:
"""Add reporting boolean to station if available"""
"""Add reporting boolean to station if available."""
good = load_stations(GOOD_PATH)
for code in stations:
stations[code]["reporting"] = code in good
return stations


def check_local_icaos() -> None:
"""Load local ICAO file if available. Not included in distro"""
"""Load local ICAO file if available. Not included in distro."""
icao_path = _FILE_DIR.parent.parent / "data" / "icaos.json"
if not icao_path.exists():
return
_SOURCE["icaos"] = icao_path.open().read().strip()


def check_local_awos() -> None:
"""Load local AWOS file if available. Not included in distro"""
"""Load local AWOS file if available. Not included in distro."""
awos_path = _FILE_DIR.parent.parent / "data" / "awos.json"
if not awos_path.exists():
return
_SOURCE["awos"] = awos_path.open().read().strip()


def download_source_files() -> bool:
"""Returns True if source files updated successfully"""
"""Return True if source files updated successfully."""
for key, route in _SOURCES.items():
resp = httpx.get(route)
if resp.status_code != 200:
Expand All @@ -234,20 +237,20 @@ def download_source_files() -> bool:


def update_station_info_date() -> None:
"""Update the package's station meta date"""
"""Update the package's station meta date."""
meta_path = _FILE_DIR.parent / "station" / "meta.py"
meta = meta_path.open().read()
target = '__LAST_UPDATED__ = "'
start = meta.find(target) + len(target)
prefix = meta[:start]
end = start + 10
output = prefix + date.today().strftime(r"%Y-%m-%d") + meta[end:]
output = prefix + datetime.now(tz=timezone.utc).date().strftime(r"%Y-%m-%d") + meta[end:]
with meta_path.open("w") as out:
out.write(output)


def save_station_data(stations: dict) -> None:
"""Save stations to JSON package data"""
"""Save stations to JSON package data."""
json.dump(
stations,
OUTPUT_PATH.open("w", encoding="utf8"),
Expand All @@ -258,7 +261,7 @@ def save_station_data(stations: dict) -> None:


def main() -> int:
"""Build/update the stations.json main file"""
"""Build/update the stations.json main file."""
LOG.info("Fetching")
if not download_source_files():
LOG.error("Unable to update source files")
Expand Down
6 changes: 3 additions & 3 deletions avwx/data/mappers.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
"""
Type mappings
"""
"""Type mappings."""

# ruff: noqa: RUF001

# Runway surface type matching with counts (2020-06)
# Uniques with < 10 occurrences omitted
Expand Down
Loading

0 comments on commit d44d949

Please sign in to comment.