Skip to content

Commit

Permalink
ci: use pyproject.toml
Browse files Browse the repository at this point in the history
  • Loading branch information
vimt committed Jun 1, 2024
1 parent c8b5e65 commit 805b820
Show file tree
Hide file tree
Showing 10 changed files with 139 additions and 109 deletions.
21 changes: 9 additions & 12 deletions .github/workflows/python.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,32 +20,29 @@ jobs:
steps:
- uses: actions/checkout@v4
- name: Setup Go environment
uses: actions/setup-go@v5.0.1
uses: actions/setup-go@v5
with:
go-version: '1.22'
- name: Setup Java JDK
uses: actions/setup-java@v4.2.1
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '22'
distribution: temurin
java-version: 22
cache: maven
- name: Set up Python 3.10
uses: actions/setup-python@v3
uses: actions/setup-python@v5
with:
python-version: "3.10"
python-version: '3.10'
cache: pip
- name: Install dependencies
run: |
sudo add-apt-repository ppa:maxmind/ppa
sudo apt update
sudo apt install libmaxminddb0 libmaxminddb-dev
python -m pip install --upgrade pip
pip install ".[test]" ruff pytest-cov
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
python -V
pip install ".[test,dev]"
- name: Check with ruff
run: |
ruff check
ruff format --check
- name: Test with pytest
run: |
pytest tests/*
pytest
10 changes: 10 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,13 @@
- [MaxMind-DB-Writer-python](#maxmind-db-writer-python)
* [Install](#install)
* [Usage](#usage)
* [Examples](#examples)
* [Using the Java Client](#using-the-java-client)
+ [TLDR](#tldr)
+ [Underlying Principles](#underlying-principles)
* [Type Enforcement](#type-enforcement)
* [Reference:](#reference-)

# MaxMind-DB-Writer-python

Make `mmdb` format ip library file which can be read by [`maxmind` official language reader](https://dev.maxmind.com/geoip/geoip2/downloadable/)
Expand Down
1 change: 0 additions & 1 deletion examples/__init__.py
Original file line number Diff line number Diff line change
@@ -1 +0,0 @@
# coding: utf-8
3 changes: 1 addition & 2 deletions examples/csv_to_mmdb.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
# coding: utf-8
import csv
from collections import defaultdict

Expand All @@ -14,7 +13,7 @@ def main():
data = defaultdict(list)

# merge cidr
with open("fake_ip_info.csv", "r") as f:
with open("fake_ip_info.csv") as f:
reader = csv.DictReader(f)
for line in reader:
data[(line["country"], line["isp"])].append(
Expand Down
85 changes: 42 additions & 43 deletions mmdb_writer.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
# coding: utf-8
__version__ = "0.2.1"

import logging
Expand All @@ -7,12 +6,12 @@
import time
from decimal import Decimal
from enum import IntEnum
from typing import Union, List, Dict, Literal
from typing import Dict, List, Literal, Union

from netaddr import IPSet, IPNetwork
from netaddr import IPNetwork, IPSet


class MmdbBaseType(object):
class MmdbBaseType:
def __init__(self, value):
self.value = value

Expand Down Expand Up @@ -97,7 +96,7 @@ class MMDBTypeID(IntEnum):
UINT64_MAX = 0xFFFFFFFFFFFFFFFF


class SearchTreeNode(object):
class SearchTreeNode:
def __init__(self, left=None, right=None):
self.left = left
self.right = right
Expand All @@ -123,12 +122,12 @@ def __setitem__(self, key, value):
self.right = value


class SearchTreeLeaf(object):
class SearchTreeLeaf:
def __init__(self, value):
self.value = value

def __repr__(self):
return "SearchTreeLeaf(value={value})".format(value=self.value)
return f"SearchTreeLeaf(value={self.value})"

__str__ = __repr__

Expand Down Expand Up @@ -156,7 +155,7 @@ def __repr__(self):
FloatType = Union[Literal["f32", "f64", "float32", "float64"] | MmdbF32 | MmdbF64]


class Encoder(object):
class Encoder:
def __init__(
self, cache=True, int_type: IntType = "auto", float_type: FloatType = "f64"
):
Expand Down Expand Up @@ -224,7 +223,8 @@ def _encode_uint(self, type_id, max_len):
def _encode_unsigned_value(value):
if value < 0 or value >= value_max:
raise ValueError(
f"encode uint{max_len * 8} fail: {value} not in range(0, {value_max})"
f"encode uint{max_len * 8} fail: "
f"{value} not in range(0, {value_max})"
)
res = b""
while value != 0 and len(res) < max_len:
Expand Down Expand Up @@ -353,7 +353,7 @@ def python_type_id(self, value):
raise ValueError(f"unknown float_type={self.float_type}")
elif value_type is Decimal:
return MMDBTypeID.DOUBLE
raise TypeError("unknown type {value_type}".format(value_type=value_type))
raise TypeError(f"unknown type {value_type}")

def encode_meta(self, meta):
res = self._make_header(MMDBTypeID.MAP, len(meta))
Expand Down Expand Up @@ -383,8 +383,8 @@ def encode(self, value, type_id=None):

try:
encoder = self.type_encoder[type_id]
except KeyError:
raise ValueError("unknown type_id={type_id}".format(type_id=type_id))
except KeyError as err:
raise ValueError(f"unknown type_id={type_id}") from err

if isinstance(value, MmdbBaseType):
value = value.value
Expand All @@ -406,7 +406,7 @@ def encode(self, value, type_id=None):
return res


class TreeWriter(object):
class TreeWriter:
encoder_cls = Encoder

def __init__(
Expand Down Expand Up @@ -541,7 +541,7 @@ def bits_rstrip(n, length=None, keep=0):
return map(int, bin(n)[2:].rjust(length, "0")[:keep])


class MMDBWriter(object):
class MMDBWriter:
def __init__(
self,
ip_version=4,
Expand All @@ -554,18 +554,20 @@ def __init__(
):
"""
Args:
ip_version (int, optional): The IP version of the database. Defaults to 4.
database_type (str, optional): The type of the database. Defaults to "GeoIP".
languages (List[str], optional): A list of languages. Defaults to [].
description (Union[Dict[str, str], str], optional): A description of the database for every language.
ipv4_compatible (bool, optional): Whether the database is compatible with IPv4. Defaults to False.
int_type (Union[str, MmdbU16, MmdbU32, MmdbU64, MmdbU128, MmdbI32], optional): The type of integer to use. Defaults to "auto".
float_type (Union[str, MmdbF32, MmdbF64], optional): The type of float to use. Defaults to "f64".
ip_version: The IP version of the database. Defaults to 4.
database_type: The type of the database. Defaults to "GeoIP".
languages: A list of languages. Defaults to [].
description: A description of the database for every language.
ipv4_compatible: Whether the database is compatible with IPv4.
int_type: The type of integer to use. Defaults to "auto".
float_type: The type of float to use. Defaults to "f64".
Note:
If you want to store an IPv4 address in an IPv6 database, you should set ipv4_compatible=True.
If you want to store an IPv4 address in an IPv6 database, you should set
ipv4_compatible=True.
If you want to use a specific integer type, you can set int_type to "u16", "u32", "u64", "u128", or "i32".
If you want to use a specific integer type, you can set int_type to
"u16", "u32", "u64", "u128", or "i32".
"""
self.tree = SearchTreeNode()
self.ipv4_compatible = ipv4_compatible
Expand All @@ -582,16 +584,12 @@ def __init__(
self._bit_length = 128 if ip_version == 6 else 32

if ip_version not in [4, 6]:
raise ValueError(
"ip_version should be 4 or 6, {} is incorrect".format(ip_version)
)
raise ValueError(f"ip_version should be 4 or 6, {ip_version} is incorrect")
if ip_version == 4 and ipv4_compatible:
raise ValueError("ipv4_compatible=True can set when ip_version=6")
if not self.binary_format_major_version:
raise ValueError(
"major_version can't be empty or 0: {}".format(
self.binary_format_major_version
)
f"major_version can't be empty or 0: {self.binary_format_major_version}"
)
if isinstance(description, str):
self.description = {i: description for i in languages}
Expand All @@ -602,22 +600,22 @@ def __init__(
self.int_type = int_type
self.float_type = float_type

def insert_network(
self, network: IPSet, content: MMDBType, overwrite=True, python_type_id_map=None
):
def insert_network(self, network: IPSet, content: MMDBType):
"""
Inserts a network into the MaxMind database.
Args:
network (IPSet): The network to be inserted. It should be an instance of netaddr.IPSet.
content (MMDBType): The content associated with the network. It can be a dictionary, list, string, bytes, integer, or boolean.
overwrite (bool, optional): If True, existing network data will be overwritten. Defaults to True.
python_type_id_map: abc
network: The network to be inserted. It should be an instance of
netaddr.IPSet.
content: The content associated with the network. It can be a
dictionary, list, string, bytes, integer, or boolean.
Raises:
ValueError: If the network is not an instance of netaddr.IPSet.
ValueError: If an IPv6 address is inserted into an IPv4-only database.
ValueError: If an IPv4 address is inserted into an IPv6 database without setting ipv4_compatible=True.
ValueError: If an IPv4 address is inserted into an IPv6 database without
setting ipv4_compatible=True.
Note:
This method modifies the internal tree structure of the MMDBWriter instance.
Expand All @@ -629,15 +627,14 @@ def insert_network(
for cidr in network:
if self.ip_version == 4 and cidr.version == 6:
raise ValueError(
"You inserted a IPv6 address {} "
"to an IPv4-only database.".format(cidr)
f"You inserted a IPv6 address {cidr} " "to an IPv4-only database."
)
if self.ip_version == 6 and cidr.version == 4:
if not self.ipv4_compatible:
raise ValueError(
"You inserted a IPv4 address {} to an IPv6 database."
f"You inserted a IPv4 address {cidr} to an IPv6 database."
"Please use ipv4_compatible=True option store "
"IPv4 address in IPv6 database as ::/96 format".format(cidr)
"IPv4 address in IPv6 database as ::/96 format"
)
cidr = cidr.ipv6(True)
node = self.tree
Expand All @@ -661,15 +658,17 @@ def insert_network(
)
)
logger.info(
f"Inserting {cidr} ({content}) into subnet of {current_cidr} ({current_node.value})"
f"Inserting {cidr} ({content}) into subnet of "
f"{current_cidr} ({current_node.value})"
)
supernet_leaf = current_node
current_node = SearchTreeNode()
previous_node[ip_bit] = current_node

if supernet_leaf:
next_bit = bits[index + 1]
# Insert supernet information on each inverse bit of the current subnet
# Insert supernet information on each inverse bit of
# the current subnet
current_node[1 - next_bit] = supernet_leaf
current_node[bits[-1]] = leaf

Expand Down
73 changes: 73 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
[build-system]
requires = ["flit_core >=3.2,<4"]
build-backend = "flit_core.buildapi"

[project]
name = "mmdb_writer"
description = "Make `mmdb` format ip library file which can be read by maxmind official language reader"
readme = "README.md"
license = {file = "LICENSE"}
requires-python = ">=3.6"
keywords = ["mmdb", "maxmind"]
authors = [{ name = "VimT", email = "me@vimt.me" } ]
classifiers = [
"Development Status :: 5 - Production/Stable",
"Intended Audience :: Developers",
"License :: OSI Approved :: MIT License",
"Natural Language :: English",
"Operating System :: OS Independent",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: Implementation :: CPython",
"Programming Language :: Python :: Implementation :: PyPy",
"Topic :: Software Development :: Build Tools",
]
dependencies = [
"netaddr>=0.7"
]
dynamic = ["version"]

[project.optional-dependencies]
test = [
"pytest >=2.7.3",
"pytest-cov",
"numpy",
"maxminddb>=1.5",
]
dev = [
"ruff"
]

[project.urls]
Home = "https://github.com/vimt/MaxMind-DB-Writer-python"
Source = "https://github.com/vimt/MaxMind-DB-Writer-python"
Tracker = "https://github.com/vimt/MaxMind-DB-Writer-python/issues"

[tool.flit.sdist]
include = ["mmdb_writer.py"]

[tool.pytest.ini_options]
testpaths = ["tests"]
filterwarnings = [
"error",
]

[tool.ruff]
fix = true
show-fixes = true
output-format = "full"

[tool.ruff.lint]
select = [
"B", # flake8-bugbear
"E", # pycodestyle error
"F", # pyflakes
"I", # isort
"UP", # pyupgrade
"W", # pycodestyle warning
]
Loading

0 comments on commit 805b820

Please sign in to comment.