From ce68d3e41981ee8ee3adde9d3b61667c033ee0fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Sat, 13 Jun 2020 20:07:07 +0200 Subject: [PATCH] cleanup legacy python checks and associated special cases --- .coveragerc | 1 - .pep8speaks.yml | 1 - setup.cfg | 1 - .../construction_data_containers.py | 21 +- yt/fields/tests/test_fields_plugins.py | 8 +- yt/frontends/exodus_ii/util.py | 8 +- yt/frontends/sdf/data_structures.py | 3 - yt/frontends/tipsy/data_structures.py | 4 - yt/funcs.py | 3 +- yt/utilities/io_handler.py | 4 +- yt/utilities/lodgeit.py | 11 +- yt/utilities/lru_cache.py | 188 ------------------ yt/utilities/poster/streaminghttp.py | 23 +-- yt/utilities/tests/test_config.py | 9 +- yt/visualization/plot_window.py | 5 +- 15 files changed, 25 insertions(+), 265 deletions(-) delete mode 100644 yt/utilities/lru_cache.py diff --git a/.coveragerc b/.coveragerc index 1a4aca30108..2dea13250ef 100644 --- a/.coveragerc +++ b/.coveragerc @@ -13,7 +13,6 @@ omit=*.yml yt/mods.py yt/utilities/fits_image.py yt/utilities/lodgeit.py - yt/utilities/lru_cache.py yt/utilities/poster/* yt/visualization/_mpl_imports.py diff --git a/.pep8speaks.yml b/.pep8speaks.yml index c225b1cd6e6..3538cb33772 100644 --- a/.pep8speaks.yml +++ b/.pep8speaks.yml @@ -75,7 +75,6 @@ pycodestyle: - \*/__config__.py - yt/visualization/_mpl_imports.py - yt/utilities/lodgeit.py - - yt/utilities/lru_cache.py - yt/utilities/poster/\* - yt/extern/\* - yt/mods.py diff --git a/setup.cfg b/setup.cfg index 6a80be350ae..a6db023776d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,7 +25,6 @@ exclude = doc, yt/visualization/_mpl_imports.py, yt/utilities/fits_image.py, yt/utilities/lodgeit.py, - yt/utilities/lru_cache.py, yt/mods.py, yt/visualization/_colormap_data.py, diff --git a/yt/data_objects/construction_data_containers.py b/yt/data_objects/construction_data_containers.py index 868631fc345..03635a90c20 100644 --- a/yt/data_objects/construction_data_containers.py +++ b/yt/data_objects/construction_data_containers.py @@ -1,7 +1,6 @@ import fileinput import io import os -import sys import zipfile from functools import wraps from re import finditer @@ -1878,18 +1877,16 @@ def _color_samples_obj( em = np.log10(em) if color_field is not None: if color_field_min is None: - if sys.version_info > (3,): - cs = [float(field) for field in cs] - cs = np.array(cs) + cs = [float(field) for field in cs] + cs = np.array(cs) mi = cs.min() else: mi = color_field_min if color_log: mi = np.log10(mi) if color_field_max is None: - if sys.version_info > (3,): - cs = [float(field) for field in cs] - cs = np.array(cs) + cs = [float(field) for field in cs] + cs = np.array(cs) ma = cs.max() else: ma = color_field_max @@ -1907,18 +1904,16 @@ def _color_samples_obj( # now, get emission if emit_field is not None: if emit_field_min is None: - if sys.version_info > (3,): - em = [float(field) for field in em] - em = np.array(em) + em = [float(field) for field in em] + em = np.array(em) emi = em.min() else: emi = emit_field_min if emit_log: emi = np.log10(emi) if emit_field_max is None: - if sys.version_info > (3,): - em = [float(field) for field in em] - em = np.array(em) + em = [float(field) for field in em] + em = np.array(em) ema = em.max() else: ema = emit_field_max diff --git a/yt/fields/tests/test_fields_plugins.py b/yt/fields/tests/test_fields_plugins.py index 4b7431f251f..0cdacfd25a6 100644 --- a/yt/fields/tests/test_fields_plugins.py +++ b/yt/fields/tests/test_fields_plugins.py @@ -1,5 +1,4 @@ import os -import sys import unittest import yt @@ -75,12 +74,9 @@ def testCustomField(self): plugin_file = os.path.join(CONFIG_DIR, ytcfg.get("yt", "pluginfilename")) msg = "INFO:yt:Loading plugins from %s" % plugin_file - if sys.version_info >= (3, 4, 0): - with self.assertLogs("yt", level="INFO") as cm: - yt.enable_plugins() - self.assertEqual(cm.output, [msg]) - else: + with self.assertLogs("yt", level="INFO") as cm: yt.enable_plugins() + self.assertEqual(cm.output, [msg]) ds = fake_random_ds(16) dd = ds.all_data() diff --git a/yt/frontends/exodus_ii/util.py b/yt/frontends/exodus_ii/util.py index 307d32edcc0..7ce4464bb60 100644 --- a/yt/frontends/exodus_ii/util.py +++ b/yt/frontends/exodus_ii/util.py @@ -1,13 +1,10 @@ import re import string -import sys from collections import OrderedDict from itertools import takewhile import numpy as np -_printable = set([ord(_) for _ in string.printable]) - def get_num_pseudo_dims(coords): D = coords.shape[1] @@ -15,9 +12,8 @@ def get_num_pseudo_dims(coords): def sanitize_string(s): - if sys.version_info > (3,): - return "".join([chr(_) for _ in takewhile(lambda a: a in _printable, s)]) - return "".join([_ for _ in takewhile(lambda a: a in string.printable, s)]) + _printable = set([ord(_) for _ in string.printable]) + return "".join([chr(_) for _ in takewhile(lambda a: a in _printable, s)]) def load_info_records(info_records): diff --git a/yt/frontends/sdf/data_structures.py b/yt/frontends/sdf/data_structures.py index 4bdc48b8e40..0466a4556bb 100644 --- a/yt/frontends/sdf/data_structures.py +++ b/yt/frontends/sdf/data_structures.py @@ -1,6 +1,5 @@ import contextlib import os -import sys import numpy as np @@ -15,8 +14,6 @@ @contextlib.contextmanager def safeopen(*args, **kwargs): - if sys.version[0] != "3": - kwargs.pop("encoding") with open(*args, **kwargs) as f: yield f diff --git a/yt/frontends/tipsy/data_structures.py b/yt/frontends/tipsy/data_structures.py index b90899f1c39..cf1f2f8db43 100644 --- a/yt/frontends/tipsy/data_structures.py +++ b/yt/frontends/tipsy/data_structures.py @@ -1,7 +1,6 @@ import glob import os import struct -import sys import numpy as np @@ -14,9 +13,6 @@ from .fields import TipsyFieldInfo -if sys.version_info > (3,): - long = int - class TipsyFile(ParticleFile): def __init__(self, ds, io, filename, file_id, range=None): diff --git a/yt/funcs.py b/yt/funcs.py index c69ff1cfdd7..bd8d27456fd 100644 --- a/yt/funcs.py +++ b/yt/funcs.py @@ -18,7 +18,7 @@ import urllib.parse import urllib.request import warnings -from functools import wraps +from functools import lru_cache, wraps from math import ceil, floor from numbers import Number as numeric_type @@ -29,7 +29,6 @@ from yt.units import YTArray, YTQuantity from yt.utilities.exceptions import YTInvalidWidthError from yt.utilities.logger import ytLogger as mylog -from yt.utilities.lru_cache import lru_cache # Some functions for handling sequences and other types diff --git a/yt/utilities/io_handler.py b/yt/utilities/io_handler.py index d759f9ba879..01dfe17acce 100644 --- a/yt/utilities/io_handler.py +++ b/yt/utilities/io_handler.py @@ -1,11 +1,11 @@ import os from collections import defaultdict from contextlib import contextmanager +from functools import _make_key, lru_cache import numpy as np from yt.geometry.selection_routines import GridSelector -from yt.utilities.lru_cache import _make_key, local_lru_cache from yt.utilities.on_demand_imports import _h5py as h5py io_registry = {} @@ -25,7 +25,7 @@ def __init__(cls, name, b, d): if hasattr(cls, "_dataset_type"): io_registry[cls._dataset_type] = cls if use_caching and hasattr(cls, "_read_obj_field"): - cls._read_obj_field = local_lru_cache( + cls._read_obj_field = lru_cache( maxsize=use_caching, typed=True, make_key=_make_io_key )(cls._read_obj_field) diff --git a/yt/utilities/lodgeit.py b/yt/utilities/lodgeit.py index 010c7f98233..3d5d8e4e20a 100644 --- a/yt/utilities/lodgeit.py +++ b/yt/utilities/lodgeit.py @@ -31,9 +31,6 @@ import sys from optparse import OptionParser -if sys.version_info >= (3, 0, 0): - unicode = str - SCRIPT_NAME = os.path.basename(sys.argv[0]) VERSION = "0.3" SERVICE_URL = "http://paste.yt-project.org/" @@ -92,14 +89,14 @@ def load_default_settings(): def make_utf8(text, encoding): """Convert a text to UTF-8, brute-force.""" try: - u = unicode(text, "utf-8") + u = str(text, "utf-8") uenc = "utf-8" except UnicodeError: try: - u = unicode(text, encoding) + u = str(text, encoding) uenc = "utf-8" except UnicodeError: - u = unicode(text, "iso-8859-15", "ignore") + u = str(text, "iso-8859-15", "ignore") uenc = "iso-8859-15" try: import chardet @@ -108,7 +105,7 @@ def make_utf8(text, encoding): d = chardet.detect(text) if d["encoding"] == uenc: return u.encode("utf-8") - return unicode(text, d["encoding"], "ignore").encode("utf-8") + return str(text, d["encoding"], "ignore").encode("utf-8") def get_xmlrpc_service(): diff --git a/yt/utilities/lru_cache.py b/yt/utilities/lru_cache.py deleted file mode 100644 index 71ee1d6fcbe..00000000000 --- a/yt/utilities/lru_cache.py +++ /dev/null @@ -1,188 +0,0 @@ -import sys -from collections import namedtuple -from functools import update_wrapper -from threading import RLock - -_CacheInfo = namedtuple("CacheInfo", ["hits", "misses", "maxsize", "currsize"]) - - -class _HashedSeq(list): - __slots__ = "hashvalue" - - def __init__(self, tup, hash=hash): - self[:] = tup - self.hashvalue = hash(tup) - - def __hash__(self): - return self.hashvalue - - -def _make_key( - args, - kwds, - typed, - kwd_mark=(object(),), - fasttypes=set((int, str, frozenset, type(None))), - sorted=sorted, - tuple=tuple, - type=type, - len=len, -): - "Make a cache key from optionally typed positional and keyword arguments" - key = args - if kwds: - sorted_items = sorted(kwds.items()) - key += kwd_mark - for item in sorted_items: - key += item - if typed: - key += tuple(type(v) for v in args) - if kwds: - key += tuple(type(v) for k, v in sorted_items) - elif len(key) == 1 and type(key[0]) in fasttypes: - return key[0] - return _HashedSeq(key) - - -def lru_cache(maxsize=100, typed=False, make_key=_make_key): - """Least-recently-used cache decorator. - If *maxsize* is set to None, the LRU features are disabled and the cache - can grow without bound. - If *typed* is True, arguments of different types will be cached separately. - For example, f(3.0) and f(3) will be treated as distinct calls with - distinct results. - Arguments to the cached function must be hashable. - View the cache statistics named tuple (hits, misses, maxsize, currsize) with - f.cache_info(). Clear the cache and statistics with f.cache_clear(). - Access the underlying function with f.__wrapped__. - See: - """ - - # Users should only access the lru_cache through its public API: - # cache_info, cache_clear, and f.__wrapped__ - # The internals of the lru_cache are encapsulated for thread safety and - # to allow the implementation to change (including a possible C version). - - def decorating_function(user_function): - - cache = dict() - stats = [0, 0] # make statistics updateable non-locally - HITS, MISSES = 0, 1 # names for the stats fields - cache_get = cache.get # bound method to lookup key or return None - _len = len # localize the global len() function - lock = RLock() # because linkedlist updates aren't threadsafe - root = [] # root of the circular doubly linked list - root[:] = [root, root, None, None] # initialize by pointing to self - nonlocal_root = [root] # make updateable non-locally - PREV, NEXT, KEY, RESULT = 0, 1, 2, 3 # names for the link fields - - if maxsize == 0: - - def wrapper(*args, **kwds): - # no caching, just do a statistics update after a successful call - result = user_function(*args, **kwds) - stats[MISSES] += 1 - return result - - elif maxsize is None: - - def wrapper(*args, **kwds): - # simple caching without ordering or size limit - key = make_key(args, kwds, typed) - result = cache_get( - key, root - ) # root used here as a unique not-found sentinel - if result is not root: - stats[HITS] += 1 - return result - result = user_function(*args, **kwds) - cache[key] = result - stats[MISSES] += 1 - return result - - else: - - def wrapper(*args, **kwds): - # size limited caching that tracks accesses by recency - try: - key = make_key(args, kwds, typed) if kwds or typed else args - except TypeError: - stats[MISSES] += 1 - return user_function(*args, **kwds) - with lock: - link = cache_get(key) - if link is not None: - # record recent use of the key by moving it to the front of the list - (root,) = nonlocal_root - link_prev, link_next, key, result = link - link_prev[NEXT] = link_next - link_next[PREV] = link_prev - last = root[PREV] - last[NEXT] = root[PREV] = link - link[PREV] = last - link[NEXT] = root - stats[HITS] += 1 - return result - result = user_function(*args, **kwds) - with lock: - (root,) = nonlocal_root - if key in cache: - # getting here means that this same key was added to the - # cache while the lock was released. since the link - # update is already done, we need only return the - # computed result and update the count of misses. - pass - elif _len(cache) >= maxsize: - # use the old root to store the new key and result - oldroot = root - oldroot[KEY] = key - oldroot[RESULT] = result - # empty the oldest link and make it the new root - root = nonlocal_root[0] = oldroot[NEXT] - oldkey = root[KEY] - oldvalue = root[RESULT] - root[KEY] = root[RESULT] = None - # now update the cache dictionary for the new links - del cache[oldkey] - cache[key] = oldroot - else: - # put result in a new link at the front of the list - last = root[PREV] - link = [last, root, key, result] - last[NEXT] = root[PREV] = cache[key] = link - stats[MISSES] += 1 - return result - - def cache_info(): - """Report cache statistics""" - with lock: - return _CacheInfo(stats[HITS], stats[MISSES], maxsize, len(cache)) - - def cache_clear(): - """Clear the cache and cache statistics""" - with lock: - cache.clear() - root = nonlocal_root[0] - root[:] = [root, root, None, None] - stats[:] = [0, 0] - - wrapper.__wrapped__ = user_function - wrapper.cache_info = cache_info - wrapper.cache_clear = cache_clear - return update_wrapper(wrapper, user_function) - - return decorating_function - - -### End of backported lru_cache - -local_lru_cache = lru_cache - -if sys.version_info[:2] >= (3, 3): - # 3.2 has an lru_cache with an incompatible API - from functools import lru_cache - -try: - from fastcache import clru_cache as lru_cache -except ImportError: - pass diff --git a/yt/utilities/poster/streaminghttp.py b/yt/utilities/poster/streaminghttp.py index 2d60fe5ba43..24443094017 100644 --- a/yt/utilities/poster/streaminghttp.py +++ b/yt/utilities/poster/streaminghttp.py @@ -32,21 +32,6 @@ import sys import urllib - -def request_has_data(req): - if sys.version_info >= (3, 0, 0): - return hasattr(req, "data") - else: - return req.has_data() - - -def request_get_data(req): - if sys.version_info >= (3, 0, 0): - return req.data - else: - return req.get_data() - - __all__ = [ "StreamingHTTPConnection", "StreamingHTTPRedirectHandler", @@ -182,8 +167,8 @@ def http_request(self, req): if we're using an iterable value""" # Make sure that if we're using an iterable object as the request # body, that we've also specified Content-Length - if request_has_data(req): - data = request_get_data(req) + if hasattr(req, "data"): + data = req.data if hasattr(data, "read") or hasattr(data, "next"): if not req.has_header("Content-length"): raise ValueError("No Content-Length specified for iterable body") @@ -208,8 +193,8 @@ def https_open(self, req): def https_request(self, req): # Make sure that if we're using an iterable object as the request # body, that we've also specified Content-Length - if request_has_data(req): - data = request_get_data(req) + if hasattr(req, "data")(req): + data = req.data if hasattr(data, "read") or hasattr(data, "next"): if not req.has_header("Content-length"): raise ValueError( diff --git a/yt/utilities/tests/test_config.py b/yt/utilities/tests/test_config.py index d8744a10358..9e5d0a0ca5f 100644 --- a/yt/utilities/tests/test_config.py +++ b/yt/utilities/tests/test_config.py @@ -2,6 +2,7 @@ import os import sys import unittest +import unittest.mock as mock from configparser import NoOptionError from io import StringIO @@ -10,14 +11,6 @@ from yt.config import _OLD_CONFIG_FILE, CONFIG_DIR, CURRENT_CONFIG_FILE, YTConfigParser from yt.fields.tests.test_fields_plugins import TEST_PLUGIN_FILE -if sys.version_info.major < 3: - try: - import mock - except ImportError: - mock = None -else: - import unittest.mock as mock - _TEST_PLUGIN = "_test_plugin.py" _DUMMY_CFG = ["[yt]", "loglevel = 49", "pluginfilename = " + _TEST_PLUGIN] diff --git a/yt/visualization/plot_window.py b/yt/visualization/plot_window.py index 010b4ec1b27..fb2799fa715 100644 --- a/yt/visualization/plot_window.py +++ b/yt/visualization/plot_window.py @@ -57,10 +57,7 @@ # Some magic for dealing with pyparsing being included or not # included in matplotlib (not in gentoo, yes in everything else) try: - if sys.version_info[0] == 3: - from matplotlib.pyparsing_py3 import ParseFatalException - else: - from matplotlib.pyparsing_py2 import ParseFatalException + from matplotlib.pyparsing_py3 import ParseFatalException except ImportError: from pyparsing import ParseFatalException