diff --git a/README.md b/README.md index ca3111d..717edc6 100644 --- a/README.md +++ b/README.md @@ -41,8 +41,9 @@ default, one can do `{..., "{{_}}": "{{unexpected}}"}` or are expected beyond the ones defined. For more complex validations, KATT supports extensible validation types. -An example is the built-in "set" validation type for JSON, -which will ignore the order of an array's items, and just check for existence: +Built-in validation types: `set`, `runtime_value`, `runtime_validation`. + +`set` will ignore the order of an array's items, and just check for existence: ``` { @@ -53,10 +54,74 @@ which will ignore the order of an array's items, and just check for existence: } ``` -The above example would validate against JSON instances such as +So the above would validate against JSON instances such as `{"some_array": [1, 3, 2]}`, or `{"some_array": [3, 2, 1]}`, or even `{"some_array": [4, 3, 2, 1]}` unless we add `{{unexpected}}`. +`runtime_value` would just run code (only `erlang` and `shell` supported for now), +while having access to `ParentKey`, `Actual`, `Unexpected` and `Callbacks`, +and return the expected value and matched against the actual one. + +``` +{ + "rfc1123": { + "{{type}}": "runtime_validation", + "erlang": "list_to_binary(httpd_util:rfc1123_date(calendar:now_to_datetime(erlang:now())))" + } +} +``` + +or in array format + +``` +{ + "rfc1123": { + "{{type}}": "runtime_validation", + "erlang": ["list_to_binary(", + " httpd_util:rfc1123_date(", + " calendar:now_to_datetime(", + " erlang:now()", + ")))" + ] + } +} +``` + +`runtime_validation` would just run code (only `erlang` and `shell` supported for now), +while having access to `ParentKey`, `Actual`, `Unexpected` and `Callbacks`, +and return + +* `{pass, [{"Key", "Value"}]}` i.e. validation passed, store new param "Key" with value "Value" +* `{not_equal, {Key, Expected, Actual}}` +* `{not_equal, {Key, Expected, Actual, [{"more", "info"}]}}` + +``` +{ + "rfc1123": { + "{{type}}": "runtime_validation", + "erlang": "Expected = httpd_util:rfc1123_date(calendar:now_to_datetime(erlang:now())), case Actual =:= Expected of true -> {pass, []}; false -> {not_equal, {ParentKey, Expected, Actual}} end" + } +} +``` + +or in array format + +``` +{ + "rfc1123": { + "{{type}}": "runtime_validation", + "erlang": ["Expected = httpd_util:rfc1123_date(calendar:now_to_datetime(erlang:now())),", + "case Actual =:= Expected of", + " true ->", + " {pass, []};", + " false ->", + " {not_equal, {ParentKey, Expected, Actual}}", + "end" + ] + } +} +``` + ## Examples A simple example that will make requests to a third party server: diff --git a/src/katt_callbacks_json.erl b/src/katt_callbacks_json.erl index 7d80258..08709b3 100644 --- a/src/katt_callbacks_json.erl +++ b/src/katt_callbacks_json.erl @@ -29,6 +29,7 @@ , parse/5 , validate_body/4 , validate_type/7 + , parse_json/1 ]). %%%_* Includes ================================================================= @@ -96,13 +97,15 @@ validate_body( false = _Justcheck validate_type( true = _JustCheck - , "set" + , Type , _ParentKey , _Options , _Actual , _Unexpected , _Callbacks - ) -> + ) when Type =:= "set" orelse + Type =:= "runtime_value" orelse + Type =:= "runtime_validation" -> true; validate_type( true = _JustCheck , _Type @@ -127,6 +130,34 @@ validate_type( false = _JustCheck , Unexpected , Callbacks ); +validate_type( false = _JustCheck + , "runtime_value" + , ParentKey + , Options + , Actual + , Unexpected + , Callbacks + ) -> + katt_validate_type:validate_type_runtime_value( ParentKey + , Options + , Actual + , Unexpected + , Callbacks + ); +validate_type( false = _JustCheck + , "runtime_validation" + , ParentKey + , Options + , Actual + , Unexpected + , Callbacks + ) -> + katt_validate_type:validate_type_runtime_validation( ParentKey + , Options + , Actual + , Unexpected + , Callbacks + ); validate_type( false = _JustCheck , _Type , _ParentKey @@ -160,7 +191,12 @@ normalize_jsx([{_, _}|_] = Items0) -> || {Key, Value} <- Items0 ]), Type = proplists:get_value(?TYPE, Items1, struct), - Items = proplists:delete(?TYPE, Items1), + Items = case Type of + struct -> + Items1; + _ -> + proplists:delete(?TYPE, Items1) + end, {Type, Items}; normalize_jsx([{}] = _Items) -> {struct, []}; diff --git a/src/katt_util.erl b/src/katt_util.erl index 0dd607f..d45ec32 100644 --- a/src/katt_util.erl +++ b/src/katt_util.erl @@ -40,6 +40,8 @@ , validate/5 , enumerate/1 , external_http_request/6 + , erl_to_list/1 + , os_cmd/2 ]). %%%_* Includes ================================================================= @@ -99,7 +101,7 @@ insert_escape_quotes(Str) when is_list(Str) -> run_result_to_jsx({error, Reason, Details}) -> [ {error, true} , {reason, Reason} - , {details, list_to_binary(io_lib:format("~p", [Details]))} + , {details, list_to_binary(erl_to_list(Details))} ]; run_result_to_jsx({ PassOrFail , ScenarioFilename @@ -141,6 +143,22 @@ external_http_request(Url, Method, Hdrs, Body, Timeout, []) -> Error end. +erl_to_list(Term) -> + io_lib:format("~p", [Term]). + +os_cmd(Cmd, Env) -> + Opt = [ stream + , exit_status + , use_stdio + , stderr_to_stdout + , in + , eof + , hide + , {env, Env} + ], + Port = open_port({spawn, Cmd}, Opt), + os_cmd_result(Port, []). + %%%_* Internal ================================================================= my_float_to_list(X) when is_float(X) -> @@ -296,7 +314,7 @@ value_to_jsx(List) when is_list(List) -> ) end; value_to_jsx(Value) -> - list_to_binary(io_lib:format("~p", [Value])). + list_to_binary(erl_to_list(Value)). is_valid(ParentKey, E, A) -> case validate(ParentKey, E, A) of @@ -489,3 +507,15 @@ enumerate(L) -> lists:zip([ integer_to_list(N) || N <- lists:seq(0, length(L) - 1) ], L). + +os_cmd_result(Port, Output) -> + receive + {Port, {data, NewOutput}} -> + os_cmd_result(Port, [Output|NewOutput]); + {Port, eof} -> + port_close(Port), + receive + {Port, {exit_status, ExitStatus}} -> + {ExitStatus, lists:flatten(Output)} + end + end. diff --git a/src/katt_validate_type.erl b/src/katt_validate_type.erl index fe0c470..baaf253 100644 --- a/src/katt_validate_type.erl +++ b/src/katt_validate_type.erl @@ -26,6 +26,8 @@ %%%_* Exports ================================================================== %% API -export([ validate_type_set/5 + , validate_type_runtime_value/5 + , validate_type_runtime_validation/5 ]). %%%_* Includes ================================================================= @@ -54,6 +56,158 @@ validate_type_set( ParentKey validate_type_set(ParentKey, Options, Actual, _Unexpected, _Callbacks) -> [{not_equal, {ParentKey, Options, Actual}}]. +-spec validate_type_runtime_value( string() + , proplists:proplist() + , proplists:proplist() + , term() + , callbacks() + ) -> pass | [validation_failure()]. +validate_type_runtime_value( ParentKey + , [{"erlang", Erlang0} | _Options] + , Actual + , Unexpected + , Callbacks + ) -> + Erlang = case Erlang0 of + {array, ErlangLines} -> + string:join(lists:map(fun({_, V}) -> V end, ErlangLines), "\n"); + _ -> + Erlang0 + end ++ ".", + {Error, Expected} = + try + {ok, Tokens, _} = erl_scan:string(Erlang), + {ok, Exprs} = erl_parse:parse_exprs(Tokens), + {value, Expected0, _} = erl_eval:exprs( Exprs + , [ {'ParentKey', ParentKey} + , {'Actual', Actual} + , {'Unexpected', Unexpected} + , {'Callbacks', Callbacks}] + ), + {undefined, Expected0} + catch + C:E -> + { Erlang ++ "~n" + ++ katt_util:erl_to_list(C) ++ ":" ++ katt_util:erl_to_list(E) + , undefined + } + end, + case Error of + undefined -> + katt_util:validate(ParentKey, Expected, Actual, Unexpected, Callbacks); + _ -> + {not_equal, {ParentKey, Error, Actual}} + end; +validate_type_runtime_value( ParentKey + , [{"shell", Shell0} | _Options] + , Actual + , Unexpected + , Callbacks + ) -> + Shell = case Shell0 of + {array, ShellLines} -> + string:join(lists:map(fun({_, V}) -> V end, ShellLines), "\n"); + _ -> + Shell0 + end, + try + {0, Erlang} = katt_util:os_cmd( Shell + , [ {"KATT_PARENT_KEY", ParentKey} + , { "KATT_ACTUAL" + , io_lib:format("~p", [Actual]) + } + , { "KATT_UNEXPECTED" + , io_lib:format("~p", [Unexpected]) + } + ]), + validate_type_runtime_value( ParentKey + , [{"erlang", Erlang}] + , Actual + , Unexpected + , Callbacks + ) + catch + C:E -> + Error = Shell ++ "~n" + ++ katt_util:erl_to_list(C) ++ ":" ++ katt_util:erl_to_list(E), + {not_equal, {ParentKey, Error, Actual}} + end. + + +-spec validate_type_runtime_validation( string() + , proplists:proplist() + , proplists:proplist() + , term() + , callbacks() + ) -> pass | [validation_failure()]. +validate_type_runtime_validation( ParentKey + , [{"erlang", Erlang0} | _Options] + , Actual + , Unexpected + , Callbacks + ) -> + Erlang = case Erlang0 of + {array, ErlangLines} -> + string:join(lists:map(fun({_, V}) -> V end, ErlangLines), "\n"); + _ -> + Erlang0 + end ++ ".", + try + {ok, Tokens, _} = erl_scan:string(Erlang), + {ok, Exprs} = erl_parse:parse_exprs(Tokens), + {value, Result, _} = erl_eval:exprs( Exprs + , [ {'ParentKey', ParentKey} + , {'Actual', Actual} + , {'Unexpected', Unexpected} + , {'Callbacks', Callbacks}] + ), + Result + catch + C:E -> + Error = { Erlang ++ "~n" + ++ katt_util:erl_to_list(C) ++ ":" ++ katt_util:erl_to_list(E) + , undefined + }, + {not_equal, {ParentKey, Error, Actual}} + end; +validate_type_runtime_validation( ParentKey + , [{"shell", Shell0} | _Options] + , Actual + , Unexpected + , Callbacks + ) -> + Shell = case Shell0 of + {array, ShellLines} -> + string:join(lists:map(fun({_, V}) -> V end, ShellLines), "\n"); + _ -> + Shell0 + end, + try + {0, Erlang} = katt_util:os_cmd( Shell + , [ {"KATT_PARENT_KEY", ParentKey} + , { "KATT_ACTUAL" + , io_lib:format("~p", [Actual]) + } + , { "KATT_UNEXPECTED" + , io_lib:format("~p", [Unexpected]) + } + ]), + validate_type_runtime_validation( ParentKey + , [{"erlang", Erlang}] + , Actual + , Unexpected + , Callbacks + ) + catch + C:E -> + Error = { Shell ++ "~n" + ++ katt_util:erl_to_list(C) ++ ":" ++ katt_util:erl_to_list(E) + , undefined + }, + {not_equal, {ParentKey, Error, Actual}} + end. + + %%%_* Internal ================================================================= -spec validate_set( string() diff --git a/test/katt_run_validate_type_runtime_tests.erl b/test/katt_run_validate_type_runtime_tests.erl new file mode 100644 index 0000000..972fb40 --- /dev/null +++ b/test/katt_run_validate_type_runtime_tests.erl @@ -0,0 +1,267 @@ +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%% Copyright 2014- AUTHORS +%%% +%%% Licensed under the Apache License, Version 2.0 (the "License"); +%%% you may not use this file except in compliance with the License. +%%% You may obtain a copy of the License at +%%% +%%% http://www.apache.org/licenses/LICENSE-2.0 +%%% +%%% Unless required by applicable law or agreed to in writing, software +%%% distributed under the License is distributed on an "AS IS" BASIS, +%%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%%% See the License for the specific language governing permissions and +%%% limitations under the License. +%%% +%%% @copyright 2014- AUTHORS +%%% +%%% KATT Run - Validate Type Runtime Tests +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +-module(katt_run_validate_type_runtime_tests). + +-include_lib("eunit/include/eunit.hrl"). + +%%% Suite + +katt_test_() -> + { setup + , spawn + , fun() -> + meck:new(katt_blueprint_parse, [passthrough]), + meck:expect( katt_blueprint_parse + , file + , fun mock_katt_blueprint_parse_file/1 + ), + meck:new(katt_callbacks, [passthrough]), + meck:expect( katt_util + , external_http_request + , fun mock_lhttpc_request/6 + ) + end + , fun(_) -> + meck:unload(katt_blueprint_parse), + meck:unload(katt_callbacks) + end + , [ katt_run_with_runtime_value() + , katt_run_with_runtime_value_array() + , katt_run_with_runtime_value_shell() + , katt_run_with_runtime_validation_pass() + , katt_run_with_runtime_validation_fail() + ] + }. + +%%% Tests + +katt_run_with_runtime_value() -> + Scenario = "/mock/runtime_value.apib", + ?_assertMatch( { pass + , Scenario + , _ + , _ + , [ {_, _, _, _, pass} + ] + } + , katt:run(Scenario) + ). + +katt_run_with_runtime_value_array() -> + Scenario = "/mock/runtime_value_array.apib", + ?_assertMatch( { pass + , Scenario + , _ + , _ + , [ {_, _, _, _, pass} + ] + } + , katt:run(Scenario) + ). + +katt_run_with_runtime_value_shell() -> + Scenario = "/mock/runtime_value_shell.apib", + ?_assertMatch( { pass + , Scenario + , _ + , _ + , [ {_, _, _, _, pass} + ] + } + , katt:run(Scenario) + ). + +katt_run_with_runtime_validation_pass() -> + Scenario = "/mock/runtime_validation_pass.apib", + ?_assertMatch( { pass + , Scenario + , _ + , _ + , [ {_, _, _, _, pass} + ] + } + , katt:run(Scenario) + ). + +katt_run_with_runtime_validation_fail() -> + Scenario = "/mock/runtime_validation_fail.apib", + ?_assertMatch( { fail + , Scenario + , _ + , _ + , [ {_, _, _, _, {fail, [ { not_equal + , { "/body/{{runtime_validation}}" + , false + , true + }} + ]}} + ] + } + , katt:run(Scenario) + ). + +%%% Helpers + +%% Mock response for runtime_value test: +mock_lhttpc_request( "http://127.0.0.1/runtime_value" + , "GET" + , _ + , _ + , _Timeout + , _Options +) -> + {ok, {{200, []}, [ + {"content-type", "application/json"} + ], <<"[ + \"/\", + 1 +] +"/utf8>>}}; + +%% Mock response for runtime_value_array test: +mock_lhttpc_request( "http://127.0.0.1/runtime_value_array" + , "GET" + , _ + , _ + , _Timeout + , _Options +) -> + {ok, {{200, []}, [ + {"content-type", "application/json"} + ], <<"[ + \"/\", + 1 +] +"/utf8>>}}; + +%% Mock response for runtime_value_shell test: +mock_lhttpc_request( "http://127.0.0.1/runtime_value_shell" + , "GET" + , _ + , _ + , _Timeout + , _Options +) -> + {ok, {{200, []}, [ + {"content-type", "application/json"} + ], <<"[ + \"/\", + 1 +] +"/utf8>>}}; + +%% Mock response for runtime_validation_pass test: +mock_lhttpc_request( "http://127.0.0.1/runtime_validation_pass" + , "GET" + , _ + , _ + , _Timeout + , _Options +) -> + {ok, {{200, []}, [ + {"content-type", "application/json"} + ], <<"{ + \"any\": \"value\" +} +"/utf8>>}}; + +%% Mock response for runtime_validation_fail test: +mock_lhttpc_request( "http://127.0.0.1/runtime_validation_fail" + , "GET" + , _ + , _ + , _Timeout + , _Options +) -> + {ok, {{200, []}, [ + {"content-type", "application/json"} + ], <<" +true +"/utf8>>}}. + +mock_katt_blueprint_parse_file("/mock/runtime_value.apib") -> + katt_blueprint_parse:string( + <<"--- Comparison as runtime_value --- + +GET /runtime_value +< 200 +< Content-Type: application/json +{ + \"{{type}}\": \"runtime_value\", + \"erlang\": \"{array, [ParentKey, 1]}\" +} +"/utf8>>); + +mock_katt_blueprint_parse_file("/mock/runtime_value_array.apib") -> + katt_blueprint_parse:string( + <<"--- Comparison as runtime_value (array) --- + +GET /runtime_value_array +< 200 +< Content-Type: application/json +{ + \"{{type}}\": \"runtime_value\", + \"erlang\": [\"{ array\", + \", [ParentKey, 1]}\" + ] +} +"/utf8>>); + +mock_katt_blueprint_parse_file("/mock/runtime_value_shell.apib") -> + katt_blueprint_parse:string( + <<"--- Comparison as runtime_value (shell) --- + +GET /runtime_value_shell +< 200 +< Content-Type: application/json +{ + \"{{type}}\": \"runtime_value\", + \"shell\": \"sh -c \\\"echo '{array, [ParentKey, 1]}'\\\"\" +} +"/utf8>>); + +mock_katt_blueprint_parse_file("/mock/runtime_validation_pass.apib") -> + katt_blueprint_parse:string( + <<"--- Comparison as runtime_validation_pass --- + +GET /runtime_validation_pass +< 200 +< Content-Type: application/json +{ + \"{{type}}\": \"runtime_validation\", + \"erlang\": \"{pass, [{\\\"Param\\\", \\\"Value\\\"}]}\" +} +"/utf8>>); + +mock_katt_blueprint_parse_file("/mock/runtime_validation_fail.apib") -> + katt_blueprint_parse:string( + <<"--- Comparison as runtime_validation_fail --- + +GET /runtime_validation_fail +< 200 +< Content-Type: application/json +{ + \"{{type}}\": \"runtime_validation\", + \"erlang\": \"{not_equal, + {\\\"/body/{{runtime_validation}}\\\", false, true} + }\" +} +"/utf8>>).