From b8c3e9aed5d572bb56f0524c4e8c70c9897b16fc Mon Sep 17 00:00:00 2001 From: jeremiah-corrado <62707311+jeremiah-corrado@users.noreply.github.com> Date: Thu, 27 Jun 2024 14:37:45 -0600 Subject: [PATCH] Refactor MessageArgs (#3358) * remove binary commandMap by including binary-payload field in MsgTuple. Fix misleading indentation in serverDaemon. Remove repeated error-handling code from ServerDaemon. Signed-off-by: Jeremiah Corrado * remove commandMapArray by adding payload field to MessageArgs Signed-off-by: Jeremiah Corrado * fix messageArgs serialization bug Signed-off-by: Jeremiah Corrado * remove unecessary string.doFormat, replacing with string.format Signed-off-by: Jeremiah Corrado * remove symbols duplicated across IOCompat versions Signed-off-by: Jeremiah Corrado * start refactor of ParameterObj. Simplified argument parsing API to include 'toScalar', 'toScalarTuple', 'toScalarList', 'toScalarArray'. Other methods were removed or "deprecated". Added 'this()' indexing to msgArgs Signed-off-by: Jeremiah Corrado * remove objType enum from Message.chpl (and message.py) - resolves #2522. Only 'efunc2Msg' was using this functionality, so put a temporary fix in place until it can be refactored Signed-off-by: Jeremiah Corrado * fix dep modules wrt IOCompat changes Signed-off-by: Jeremiah Corrado * remove objType from message.py tests Signed-off-by: Jeremiah Corrado * fix tuple and array deserialization for array-api-builds. Fix a bug in the array-api 'tolist' introduced in #3242 Signed-off-by: Jeremiah Corrado * remove duplicate serialize method from MessageArgs Signed-off-by: Jeremiah Corrado * fix formatting in Message.chpl Signed-off-by: Jeremiah Corrado --------- Signed-off-by: Jeremiah Corrado Co-authored-by: ajpotts --- PROTO_tests/tests/message_test.py | 29 +- ServerModules.cfg | 2 +- arkouda/array_api/array_object.py | 2 +- arkouda/message.py | 69 ++-- dep/checkHDF5.chpl | 3 +- dep/checkZMQ.chpl | 3 +- src/AryUtil.chpl | 4 +- src/BigIntMsg.chpl | 3 +- src/CSVMsg.chpl | 3 +- src/CommandMap.chpl | 11 +- src/DataFrameIndexingMsg.chpl | 2 +- src/EfuncMsg.chpl | 24 +- src/FileIO.chpl | 5 +- src/GenSymIO.chpl | 3 +- src/HDF5Msg.chpl | 3 +- src/HashMsg.chpl | 3 +- src/IOUtils.chpl | 85 +++++ src/IndexingMsg.chpl | 31 +- src/Logging.chpl | 7 +- src/ManipulationMsg.chpl | 72 ++-- src/Message.chpl | 488 +++++++++++-------------- src/MetricsMsg.chpl | 3 +- src/MsgProcessing.chpl | 2 +- src/MultiTypeRegEntry.chpl | 5 +- src/MultiTypeSymbolTable.chpl | 11 +- src/ParquetMsg.chpl | 3 +- src/ReductionMsg.chpl | 4 +- src/RegistrationMsg.chpl | 3 +- src/Registry.chpl | 1 + src/SegmentedMsg.chpl | 3 +- src/ServerConfig.chpl | 3 +- src/ServerDaemon.chpl | 15 +- src/StatsMsg.chpl | 3 +- src/StatusMsg.chpl | 6 +- src/TimeClassMsg.chpl | 2 +- src/TransferMsg.chpl | 2 +- src/UtilMsg.chpl | 4 +- src/compat/e-132/ArkoudaIOCompat.chpl | 60 --- src/compat/eq-133/ArkoudaIOCompat.chpl | 60 --- src/compat/eq-134/ArkoudaIOCompat.chpl | 60 --- src/compat/ge-20/ArkoudaIOCompat.chpl | 60 --- tests/message_test.py | 19 +- 42 files changed, 455 insertions(+), 726 deletions(-) create mode 100644 src/IOUtils.chpl diff --git a/PROTO_tests/tests/message_test.py b/PROTO_tests/tests/message_test.py index 28a5e1fcd8..88a619a17e 100644 --- a/PROTO_tests/tests/message_test.py +++ b/PROTO_tests/tests/message_test.py @@ -104,7 +104,6 @@ def test_scalar_args(self, dtype): json.dumps( { "key": "arg1", - "objType": "VALUE", "dtype": ak.resolve_scalar_dtype(val1), "val": str(val1), } @@ -112,7 +111,6 @@ def test_scalar_args(self, dtype): json.dumps( { "key": "arg2", - "objType": "VALUE", "dtype": ak.resolve_scalar_dtype(val2), "val": str(val2), } @@ -127,7 +125,7 @@ def test_addl_str(self): expected = json.dumps( [ json.dumps( - {"key": "arg", "objType": "VALUE", "dtype": ak.resolve_scalar_dtype(val), "val": val} + {"key": "arg", "dtype": ak.resolve_scalar_dtype(val), "val": val} ), ] ) @@ -144,7 +142,6 @@ def test_list_arg(self, dtype): json.dumps( { "key": "list1", - "objType": "LIST", "dtype": ak.resolve_scalar_dtype(l1[0]), "val": json.dumps([str(x) for x in l1]), } @@ -152,7 +149,6 @@ def test_list_arg(self, dtype): json.dumps( { "key": "list2", - "objType": "LIST", "dtype": ak.resolve_scalar_dtype(l2[0]), "val": json.dumps([str(x) for x in l2]), } @@ -170,7 +166,6 @@ def test_list_addl_str(self): json.dumps( { "key": "str_list", - "objType": "LIST", "dtype": ak.resolve_scalar_dtype(string_list[0]), "val": json.dumps(string_list), } @@ -184,7 +179,7 @@ def test_datetime_arg(self): size, args = _json_args_to_str({"datetime": dt}) expected = json.dumps( - [json.dumps({"key": "datetime", "objType": "PDARRAY", "dtype": "int64", "val": dt.name})] + [json.dumps({"key": "datetime", "dtype": "int64", "val": dt.name})] ) assert args == expected @@ -193,7 +188,7 @@ def test_ip_arg(self): ip = ak.ip_address(a) size, args = _json_args_to_str({"ip": ip}) expected = json.dumps( - [json.dumps({"key": "ip", "objType": "PDARRAY", "dtype": "uint64", "val": ip.name})] + [json.dumps({"key": "ip", "dtype": "uint64", "val": ip.name})] ) assert args == expected @@ -202,7 +197,7 @@ def test_fields_arg(self): f = ak.Fields(a, names="ABCD") size, args = _json_args_to_str({"fields": f}) expected = json.dumps( - [json.dumps({"key": "fields", "objType": "PDARRAY", "dtype": "uint64", "val": f.name})] + [json.dumps({"key": "fields", "dtype": "uint64", "val": f.name})] ) assert args == expected @@ -214,10 +209,10 @@ def test_pda_arg(self, dtype): expected = json.dumps( [ json.dumps( - {"key": "pda1", "objType": "PDARRAY", "dtype": str(pda1.dtype), "val": pda1.name} + {"key": "pda1", "dtype": str(pda1.dtype), "val": pda1.name} ), json.dumps( - {"key": "pda2", "objType": "PDARRAY", "dtype": str(pda2.dtype), "val": pda2.name} + {"key": "pda2", "dtype": str(pda2.dtype), "val": pda2.name} ), ] ) @@ -229,7 +224,6 @@ def test_pda_arg(self, dtype): json.dumps( { "key": "pda_list", - "objType": "LIST", "dtype": ak.pdarray.objType, "val": json.dumps([pda1.name, pda2.name]), } @@ -244,8 +238,8 @@ def test_segstr_arg(self): size, args = _json_args_to_str({"str1": str1, "str2": str2}) expected = json.dumps( [ - json.dumps({"key": "str1", "objType": "SEGSTRING", "dtype": "str", "val": str1.name}), - json.dumps({"key": "str2", "objType": "SEGSTRING", "dtype": "str", "val": str2.name}), + json.dumps({"key": "str1", "dtype": "str", "val": str1.name}), + json.dumps({"key": "str2", "dtype": "str", "val": str2.name}), ] ) assert args == expected @@ -256,7 +250,6 @@ def test_segstr_arg(self): json.dumps( { "key": "str_list", - "objType": "LIST", "dtype": ak.Strings.objType, "val": json.dumps([str1.name, str2.name]), } @@ -280,14 +273,12 @@ def test_dict_arg(self): json.dumps( { "key": "json_1", - "objType": "DICT", "dtype": "dict", "val": json.dumps( [ json.dumps( { "key": "param1", - "objType": "VALUE", "dtype": ak.resolve_scalar_dtype(json_1["param1"]), "val": "1", } @@ -295,7 +286,6 @@ def test_dict_arg(self): json.dumps( { "key": "param2", - "objType": "VALUE", "dtype": ak.resolve_scalar_dtype(json_1["param2"]), "val": "abc", } @@ -303,7 +293,6 @@ def test_dict_arg(self): json.dumps( { "key": "param3", - "objType": "LIST", "dtype": ak.resolve_scalar_dtype(json_1["param3"][0]), "val": json.dumps([str(x) for x in json_1["param3"]]), } @@ -311,7 +300,6 @@ def test_dict_arg(self): json.dumps( { "key": "param4", - "objType": "PDARRAY", "dtype": str(json_1["param4"].dtype), "val": json_1["param4"].name, } @@ -319,7 +307,6 @@ def test_dict_arg(self): json.dumps( { "key": "param5", - "objType": "SEGSTRING", "dtype": "str", "val": json_1["param5"].name, } diff --git a/ServerModules.cfg b/ServerModules.cfg index 96b7a58cfd..cf43c05e77 100644 --- a/ServerModules.cfg +++ b/ServerModules.cfg @@ -19,7 +19,7 @@ IndexingMsg JoinEqWithDTMsg KExtremeMsg LogMsg -# ManipulationMsg +ManipulationMsg OperatorMsg ParquetMsg RandMsg diff --git a/arkouda/array_api/array_object.py b/arkouda/array_api/array_object.py index fff5d08c3a..6c03c5e552 100644 --- a/arkouda/array_api/array_object.py +++ b/arkouda/array_api/array_object.py @@ -97,7 +97,7 @@ def tolist(self): :func:`~arkouda.client.maxTransferBytes`) """ x = self._array.to_list() - if self._has_single_elem(): + if self.shape == (): # to match numpy, return a scalar for a 0-dimensional array return x[0] else: diff --git a/arkouda/message.py b/arkouda/message.py index a1f83eeac2..66e720121f 100644 --- a/arkouda/message.py +++ b/arkouda/message.py @@ -10,47 +10,15 @@ from arkouda.dtypes import isSupportedNumber, resolve_scalar_dtype -class ObjectType(Enum): - """ - Class used for assigning object types in the JSON string - sent to the server for processing - """ - - PDARRAY = "PDARRAY" - STRINGS = "SEGSTRING" - SEGARRAY = "SEGARRAY" - LIST = "LIST" - DICT = "DICT" - VALUE = "VALUE" - DATETIME = "DATETIME" - TIMEDELTA = "TIMEDELTA" - - def __str__(self) -> str: - """ - Overridden method returns value, which is useful in outputting - a MessageType object to JSON. - """ - return self.value - - def __repr__(self) -> str: - """ - Overridden method returns value, which is useful in outputting - a MessageType object to JSON. - """ - return self.value - - class ParameterObject: - __slots__ = ("key", "objType", "dtype", "val") + __slots__ = ("key", "dtype", "val") key: str - objType: MessageFormat dtype: str val: str - def __init__(self, key, objType, dtype, val): + def __init__(self, key, dtype, val): object.__setattr__(self, "key", key) - object.__setattr__(self, "objType", objType) object.__setattr__(self, "dtype", dtype) object.__setattr__(self, "val", val) @@ -58,7 +26,6 @@ def __init__(self, key, objType, dtype, val): def dict(self): return { "key": self.key, - "objType": str(self.objType), "dtype": self.dtype, "val": self.val, } @@ -80,7 +47,7 @@ def _build_pdarray_param(key: str, val) -> ParameterObject: ------- ParameterObject """ - return ParameterObject(key, ObjectType.PDARRAY, str(val.dtype), val.name) + return ParameterObject(key, str(val.dtype), val.name) @staticmethod @typechecked @@ -101,7 +68,7 @@ def _build_strings_param(key: str, val) -> ParameterObject: """ # empty string if name of String obj is none name = val.name if val.name else "" - return ParameterObject(key, ObjectType.STRINGS, "str", name) + return ParameterObject(key, "str", name) @staticmethod @typechecked @@ -121,7 +88,7 @@ def _build_segarray_param(key: str, val) -> ParameterObject: ParameterObject """ data = json.dumps({"segments": val.segments.name, "values": val.values.name}) - return ParameterObject(key, ObjectType.SEGARRAY, str(val.values.dtype), data) + return ParameterObject(key, str(val.values.dtype), data) @staticmethod def _is_supported_value(val): @@ -141,6 +108,25 @@ def _format_param(p): else p.name ) + @staticmethod + @typechecked + def _build_tuple_param(key: str, val: tuple) -> ParameterObject: + """ + Create a ParameterObject from a tuple + + Parameters + ---------- + key : str + key from the dictionary object + val : tuple + tuple object to format as string + + Returns + ------- + ParameterObject + """ + return ParameterObject._build_list_param(key, list(val)) + @staticmethod @typechecked def _build_list_param(key: str, val: list) -> ParameterObject: @@ -184,7 +170,7 @@ def _build_list_param(key: str, val: list) -> ParameterObject: str(p) if ParameterObject._is_supported_value(p) else ParameterObject._format_param(p) for p in val ] - return ParameterObject(key, ObjectType.LIST, t, json.dumps(data)) + return ParameterObject(key, t, json.dumps(data)) @staticmethod @typechecked @@ -195,7 +181,7 @@ def _build_dict_param(key: str, val: Dict) -> ParameterObject: raise TypeError(f"Argument keys are required to be str. Found {type(k)}") param = ParameterObject.factory(k, v) j.append(json.dumps(param.dict)) - return ParameterObject(key, ObjectType.DICT, str(dict.__name__), json.dumps(j)) + return ParameterObject(key, str(dict.__name__), json.dumps(j)) @staticmethod @typechecked @@ -215,7 +201,7 @@ def _build_gen_param(key: str, val) -> ParameterObject: ParameterObject """ v = val if isinstance(val, str) else str(val) - return ParameterObject(key, ObjectType.VALUE, resolve_scalar_dtype(val), v) + return ParameterObject(key, resolve_scalar_dtype(val), v) @staticmethod def generate_dispatch() -> Dict: @@ -234,6 +220,7 @@ def generate_dispatch() -> Dict: SegArray.__name__: ParameterObject._build_segarray_param, list.__name__: ParameterObject._build_list_param, dict.__name__: ParameterObject._build_dict_param, + tuple.__name__: ParameterObject._build_tuple_param, } @classmethod diff --git a/dep/checkHDF5.chpl b/dep/checkHDF5.chpl index dd8638d65f..304df3b8fa 100644 --- a/dep/checkHDF5.chpl +++ b/dep/checkHDF5.chpl @@ -1,9 +1,8 @@ use HDF5, CTypes; -use ArkoudaIOCompat; proc main() { var H5major: c_uint, H5minor: c_uint, H5micro: c_uint; C_HDF5.H5get_libversion(H5major, H5minor, H5micro); - writefCompat("Found HDF5 version: %?.%?.%?\n", H5major, H5minor, H5micro); + writef("Found HDF5 version: %?.%?.%?\n", H5major, H5minor, H5micro); return 0; } diff --git a/dep/checkZMQ.chpl b/dep/checkZMQ.chpl index 2b7f24a844..e53a89ddbc 100644 --- a/dep/checkZMQ.chpl +++ b/dep/checkZMQ.chpl @@ -1,8 +1,7 @@ use ZMQ; -use ArkoudaIOCompat; proc main() { var (Zmajor, Zminor, Zmicro) = ZMQ.version; - writefCompat("Found ZMQ version: %?.%?.%?\n", Zmajor, Zminor, Zmicro); + writef("Found ZMQ version: %?.%?.%?\n", Zmajor, Zminor, Zmicro); return 0; } diff --git a/src/AryUtil.chpl b/src/AryUtil.chpl index 419f5a69e0..1d9beec257 100644 --- a/src/AryUtil.chpl +++ b/src/AryUtil.chpl @@ -745,7 +745,7 @@ module AryUtil // ranges of flat indices owned by each locale const flatLocRanges = [loc in Locales] d.localSubdomain(loc).dim(0); - coforall loc in Locales do on loc { + coforall loc in Locales with (ref unflat) do on loc { const lduf = unflat.domain.localSubdomain(), lastRank = lduf.dim(N-1); @@ -807,7 +807,7 @@ module AryUtil // ranges of flat indices owned by each locale const flatLocRanges = [loc in Locales] flat.domain.localSubdomain(loc).dim(0); - coforall loc in Locales do on loc { + coforall loc in Locales with (ref flat) do on loc { const ld = d.localSubdomain(), lastRank = ld.dim(d.rank-1); diff --git a/src/BigIntMsg.chpl b/src/BigIntMsg.chpl index 64e243fc49..ff27c89a63 100644 --- a/src/BigIntMsg.chpl +++ b/src/BigIntMsg.chpl @@ -10,8 +10,7 @@ module BigIntMsg { use ServerErrorStrings; use BigInteger; use List; - - use ArkoudaIOCompat; + use IOUtils; private config const logLevel = ServerConfig.logLevel; private config const logChannel = ServerConfig.logChannel; diff --git a/src/CSVMsg.chpl b/src/CSVMsg.chpl index 4834a964f0..5d156ed447 100644 --- a/src/CSVMsg.chpl +++ b/src/CSVMsg.chpl @@ -16,8 +16,7 @@ module CSVMsg { use Sort; use FileIO; use Set; - - use ArkoudaIOCompat; + use IOUtils; const CSV_HEADER_OPEN = "**HEADER**"; const CSV_HEADER_CLOSE = "*/HEADER/*"; diff --git a/src/CommandMap.chpl b/src/CommandMap.chpl index 8dd145e70f..af64fdae6f 100644 --- a/src/CommandMap.chpl +++ b/src/CommandMap.chpl @@ -2,7 +2,9 @@ module CommandMap { use Message; use MultiTypeSymbolTable; - use ArkoudaIOCompat; + use JSON; + use IO; + use IOUtils; use Map; /** @@ -35,6 +37,13 @@ module CommandMap { moduleMap.add(cmd, modName); } + proc writeUsedModulesJson(ref mods: set(string)) { + const cfgFile = try! open("UsedModules.json", ioMode.cw), + w = try! cfgFile.writer(locking=false, serializer = new jsonSerializer()); + + try! w.write(mods); + } + proc writeUsedModules(fmt: string = "cfg") { select fmt { when "json" do writeUsedModulesJson(usedModules); diff --git a/src/DataFrameIndexingMsg.chpl b/src/DataFrameIndexingMsg.chpl index b477474b11..c5d414fc65 100644 --- a/src/DataFrameIndexingMsg.chpl +++ b/src/DataFrameIndexingMsg.chpl @@ -13,7 +13,7 @@ use MultiTypeSymEntry; use MultiTypeSymbolTable; - use ArkoudaIOCompat; + use IOUtils; private config const logLevel = ServerConfig.logLevel; private config const logChannel = ServerConfig.logChannel; diff --git a/src/EfuncMsg.chpl b/src/EfuncMsg.chpl index 06ea4b92cb..8d9f3c8cf3 100644 --- a/src/EfuncMsg.chpl +++ b/src/EfuncMsg.chpl @@ -498,14 +498,16 @@ module EfuncMsg var repMsg: string; var rname = st.nextName(); var efunc = msgArgs.getValueOf("func"); - var aParam = msgArgs.get("A"); - var bParam = msgArgs.get("B"); + var aParam = msgArgs["A"]; + var bParam = msgArgs["B"]; - // TODO see issue #2522: merge enum ObjType and ObjectType - select (aParam.objType, bParam.objType) { - when (ObjectType.PDARRAY, ObjectType.PDARRAY) { - var aGen: borrowed GenSymEntry = getGenericTypedArrayEntry(aParam.val, st); - var bGen: borrowed GenSymEntry = getGenericTypedArrayEntry(bParam.val, st); + const (aName, aIsPdarray) = aParam.tryGetScalar(string), + (bName, bIsPdarray) = bParam.tryGetScalar(string); + + select (aIsPdarray, bIsPdarray) { + when (true, true) { + var aGen: borrowed GenSymEntry = getGenericTypedArrayEntry(aName, st); + var bGen: borrowed GenSymEntry = getGenericTypedArrayEntry(bName, st); if aGen.shape != bGen.shape { var errorMsg = "shape mismatch in arguments to "+pn; eLogger.error(getModuleName(),getRoutineName(),getLineNumber(),errorMsg); @@ -615,8 +617,8 @@ module EfuncMsg } } } - when (ObjectType.PDARRAY, ObjectType.VALUE) { - var aGen: borrowed GenSymEntry = getGenericTypedArrayEntry(aParam.val, st); + when (true, false) { + var aGen: borrowed GenSymEntry = getGenericTypedArrayEntry(aName, st); select (aGen.dtype, bParam.getDType()) { when (DType.Int64, DType.Int64) { var aEnt = toSymEntry(aGen, int, nd); @@ -721,8 +723,8 @@ module EfuncMsg } } } - when (ObjectType.VALUE, ObjectType.PDARRAY) { - var bGen: borrowed GenSymEntry = getGenericTypedArrayEntry(bParam.val, st); + when (false, true) { + var bGen: borrowed GenSymEntry = getGenericTypedArrayEntry(bName, st); select (aParam.getDType(), bGen.dtype) { when (DType.Int64, DType.Int64) { var aScal = aParam.getIntValue(); diff --git a/src/FileIO.chpl b/src/FileIO.chpl index f583767836..8a2104ddbd 100644 --- a/src/FileIO.chpl +++ b/src/FileIO.chpl @@ -13,6 +13,7 @@ module FileIO { use Map; use ArkoudaIOCompat; + use IOUtils; use ServerConfig, Logging, CommandMap; private config const logLevel = ServerConfig.logLevel; @@ -226,7 +227,7 @@ module FileIO { var f:file = open(path, ioMode.r); var reader = fileIOReaderCompat(f); var header:bytes; - if (binaryCheckCompat(reader)) { + if reader.deserializerType == binarySerializer { reader.readBytes(header, 8); } else { throw getErrorWithContext( @@ -240,7 +241,7 @@ module FileIO { f.close(); } catch e { throw getErrorWithContext( - msg=formatString(e), + msg="%?".format(e), getLineNumber(), getRoutineName(), getModuleName(), diff --git a/src/GenSymIO.chpl b/src/GenSymIO.chpl index 70fcad02cc..a5f76c58ff 100644 --- a/src/GenSymIO.chpl +++ b/src/GenSymIO.chpl @@ -19,8 +19,7 @@ module GenSymIO { use Map; use CTypes; use CommAggregation; - - use ArkoudaIOCompat; + use IOUtils; private config const logLevel = ServerConfig.logLevel; private config const logChannel = ServerConfig.logChannel; diff --git a/src/HDF5Msg.chpl b/src/HDF5Msg.chpl index 7a9568c46f..48d8662bc3 100644 --- a/src/HDF5Msg.chpl +++ b/src/HDF5Msg.chpl @@ -27,8 +27,7 @@ module HDF5Msg { use CTypes; use BigInteger; use Regex; - - use ArkoudaIOCompat; + use IOUtils; private config const logLevel = ServerConfig.logLevel; diff --git a/src/HashMsg.chpl b/src/HashMsg.chpl index 1f95076974..057c038934 100644 --- a/src/HashMsg.chpl +++ b/src/HashMsg.chpl @@ -13,8 +13,7 @@ module HashMsg { use AryUtil; use UniqueMsg; use Map; - - use ArkoudaIOCompat; + use IOUtils; private config const logLevel = ServerConfig.logLevel; private config const logChannel = ServerConfig.logChannel; diff --git a/src/IOUtils.chpl b/src/IOUtils.chpl new file mode 100644 index 0000000000..5be14a97e9 --- /dev/null +++ b/src/IOUtils.chpl @@ -0,0 +1,85 @@ +module IOUtils { + use IO; + use JSON; + + /* + Format an argument as a JSON string + */ + proc formatJson(input): string throws { + var f = openMemFile(); + f.writer(serializer = new jsonSerializer(), locking=false).write(input); + return f.reader(locking=false).readAll(string); + } + + proc formatJson(input:string, vals...?): string throws { + var f = openMemFile(); + f.writer(serializer = new jsonSerializer(), locking=false).writef(input, (...vals)); + return f.reader(locking=false).readAll(string); + } + + /* + Deserialize a Chapel array of strings from a JSON string + */ + proc jsonToArray(json: string, type t, size: int) throws { + var f = openMemFile(); + var w = f.writer(locking=false); + w.write(json); + w.close(); + var r = f.reader(deserializer=new jsonDeserializer(), locking=false); + var array: [0..#size] t; + + // r.readf("%?", array); + + // temporary solution for tuples until frontend JSON serialization is fixed + // (i.e., the frontend should not serialize an array of numbers with quotes around each number) + var first = true; + r.matchLiteral("["); + for i in 0..= 0 && x < max then return x; if x < 0 && x >= -max then return x + max; else throw new ErrorWithContext( @@ -228,102 +323,37 @@ module Message { * Return the value as uint64 * Returns uint */ + // deprecated proc getUIntValue(): uint throws { - try { - return this.val:uint; - } - catch { - throw new owned ErrorWithContext("Parameter cannot be cast as uint. Attempting to cast %s as type uint failed".format(this.val), - getLineNumber(), - getRoutineName(), - getModuleName(), - "TypeError"); - } + return this.toScalar(uint); } + // deprecated proc getUInt8Value(): uint(8) throws { - try { - return this.val:uint(8); - } - catch { - throw new owned ErrorWithContext("Parameter cannot be cast as uint(8). Attempting to cast %s as type uint(8) failed".format(this.val), - getLineNumber(), - getRoutineName(), - getModuleName(), - "TypeError"); - } + return this.toScalar(uint(8)); } /* * Return the value as float64 * Returns real */ + // deprecated proc getRealValue(): real throws { - try { - return this.val:real; - } - catch { - throw new owned ErrorWithContext("Parameter cannot be cast as real. Attempting to cast %s as type real failed".format(this.val), - getLineNumber(), - getRoutineName(), - getModuleName(), - "TypeError"); - } + return this.toScalar(real); } /* * Return the value as bool * Returns bool */ + // deprecated proc getBoolValue(): bool throws { - try { - return this.val.toLower():bool; - } - catch { - throw new owned ErrorWithContext("Parameter cannot be cast as bool. Attempting to cast %s as type bool failed".format(this.val), - getLineNumber(), - getRoutineName(), - getModuleName(), - "TypeError"); - } + return this.toScalar(bool); } + // deprecated proc getBigIntValue(): bigint throws { - try { - return this.val:bigint; - } - catch { - throw new owned ErrorWithContext("Parameter cannot be cast as bigint. Attempting to cast %s as type bigint failed".format(this.val), - getLineNumber(), - getRoutineName(), - getModuleName(), - "TypeError"); - } - } - - /* - * Return the value as the provided type - */ - proc getValueAsType(type t = string): t throws { - if objType != ObjectType.VALUE { - throw new owned ErrorWithContext("The value provided is not a castable type, please use ParameterObj.getSymEntry for this object.", - getLineNumber(), - getRoutineName(), - getModuleName(), - "TypeError"); - } - - try { - if t == bool then return this.val.toLower():bool; - else return this.val:t; - } - catch { - throw new owned ErrorWithContext("Parameter cannot be cast as %?. Attempting to cast %s as type %? failed".format(t:string, this.val, t:string), - getLineNumber(), - getRoutineName(), - getModuleName(), - "TypeError"); - } + return this.toScalar(bigint); } /* @@ -331,66 +361,17 @@ module Message { :size: int: number of values in the list Note - not yet able to handle list of pdarray or SegString names */ + // deprecated proc getList(size: int) throws { - if this.objType != ObjectType.LIST { - throw new owned ErrorWithContext("Parameter with key, %s, is not a list.".format(this.key), - getLineNumber(), - getRoutineName(), - getModuleName(), - "TypeError"); - } - return jsonToPdArray(this.val, size); - } - - proc getListAs(type t, size: int) throws { - if this.objType != ObjectType.LIST { - throw new owned ErrorWithContext("Parameter with key, %s, is not a list.".format(this.key), - getLineNumber(), - getRoutineName(), - getModuleName(), - "TypeError"); - } - try { - const vals = jsonToPdArray(this.val, size); - var ret: [0.. s { - break; + if p.key == key { + return p; } } - return formatJson(json); + + throw new owned ErrorWithContext("JSON argument key Not Found; %s".format(key), + getLineNumber(), + getRoutineName(), + getModuleName(), + "KeyNotFound"); + } + + iter these(): ParameterObj { + for p in this.param_list { + yield p; + } } override proc serialize(writer: fileWriter(?), ref serializer: ?st) throws { @@ -462,67 +453,24 @@ module Message { * Returns ParameterObj with the provided key * Throws KeyNotFound error if the provide key does not exist. */ - proc get(key: string) throws { - for p in this.param_list { - if p.key == key { - return p; - } - } - throw new owned ErrorWithContext("Key Not Found; %s".format(key), - getLineNumber(), - getRoutineName(), - getModuleName(), - "KeyNotFound"); + proc get(key: string): ParameterObj throws { + return this[key]; } - proc getValueOf(key: string) throws { - for p in this.param_list { - if p.key == key { - return p.val; - } - } - throw new owned ErrorWithContext("Key Not Found; %s".format(key), - getLineNumber(), - getRoutineName(), - getModuleName(), - "KeyNotFound"); + proc getValueOf(key: string): string throws { + return this[key].val; } /* - Return "iterable" of ParameterObj - */ - proc items() { - return this.param_list; - } - - /* - Return a list of all keys - */ - proc keys() { - var key_list: [0..#this.size] string; - forall (idx, p) in zip(0..#this.size, this.param_list) { - key_list[idx] = p.key; - } - return key_list; - } - - /* - Return a list of all values - */ - proc vals(){ - var val_list: [0..#this.size] string; - forall (idx, p) in zip(0..#this.size, this.param_list) { - val_list[idx] = p.val; - } - return val_list; - } - - /* - Return bool if param_list contains given key name + Return true if there is an argument with the given name, false otherwise */ proc contains(key: string): bool { - var key_list = new list(this.keys()); - return key_list.contains(key); + for p in this.param_list { + if p.key == key { + return true; + } + } + return false; } } @@ -534,7 +482,8 @@ module Message { var newmem = openMemFile(); newmem.writer(locking=false).write(payload); try { - readfCompat(newmem, "%?", p); + var nreader = newmem.reader(deserializer=new jsonDeserializer(), locking=false); + nreader.readf("%?", p); } catch bfe : BadFormatError { throw new owned ErrorWithContext("Incorrect JSON format %s".format(payload), getLineNumber(), @@ -549,7 +498,7 @@ module Message { Parse arguments formatted as json string into objects */ proc parseMessageArgs(json_str: string, size: int) throws { - var pArr = jsonToPdArray(json_str, size); + var pArr = jsonToArray(json_str, string, size); var param_list = new list(ParameterObj, parSafe=true); forall j_str in pArr with (ref param_list) { param_list.pushBack(parseParameter(j_str)); @@ -567,9 +516,9 @@ module Message { proc deserialize(ref msg: RequestMsg, request: string) throws { var newmem = openMemFile(); newmem.writer(locking=false).write(request); - var nreader = newmem.reader(locking=false); try { - readfCompat(newmem, "%?", msg); + var nreader = newmem.reader(deserializer=new jsonDeserializer(), locking=false); + nreader.readf("%?", msg); } catch bfe : BadFormatError { throw new owned ErrorWithContext("Incorrect JSON format %s".format(request), getLineNumber(), @@ -578,7 +527,7 @@ module Message { "ValueError"); } } - + /* * Generates a ReplyMsg object and serializes it into a JSON-formatted reply message */ @@ -587,33 +536,4 @@ module Message { return formatJson(new MsgTuple(msg=msg,msgType=msgType, msgFormat=msgFormat, user=user)); } - - /* - * Converts the JSON array to a pdarray - */ - proc jsonToPdArray(json: string, size: int) throws { - return jsonToPdArrayCompat(json, size); - } - - /* - Helper function to parse a JSON string as a tuple of integers - */ - proc parseJsonTuple(json: string, param size: int): size*int throws { - var f = openMemFile(); - var w = f.writer(locking=false); - w.write(json); - w.close(); - var r = f.reader(locking=false), - t: size*int, - first = true; - - r.readLiteral("("); - for i in 0..