From 558b9a54bfad0af7d858703e53a4d69ee0d86715 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= Date: Wed, 8 Jul 2020 12:00:58 +0100 Subject: [PATCH 1/3] Change shebang handling In preparation for reading from stdin, we can't un-read the first line --- src/erlfmt_scan.erl | 61 +++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 33 deletions(-) diff --git a/src/erlfmt_scan.erl b/src/erlfmt_scan.erl index 1d36449f..692e0e69 100644 --- a/src/erlfmt_scan.erl +++ b/src/erlfmt_scan.erl @@ -76,40 +76,13 @@ ). -spec io_node(file:io_device()) -> node_ret(). -io_node(IO) -> node(fun read_shebang_io/1, fun io_scan_erl_node/2, IO). +io_node(IO) -> node(fun io_scan_erl_node/2, IO). -spec string_node(string()) -> node_ret(). -string_node(String) -> node(fun read_shebang_string/1, fun erl_scan_tokens/2, String). - -read_shebang_io(IO) -> - case file:read_line(IO) of - {ok, Line} -> - case read_shebang_string(Line) of - {ok, Token, "", Loc} -> - {ok, Token, IO, Loc}; - error -> - {ok, _} = file:position(IO, bof), - error - end; - _ -> - error - end. +string_node(String) -> node(fun erl_scan_tokens/2, String). -read_shebang_string("#!" ++ _ = String) -> - [Shebang, Rest] = string:split(String, "\n"), - Anno = [{text, Shebang}, {location, ?START_LOCATION}], - Token = {shebang, Anno, Shebang}, - {ok, Token, Rest, {2, 1}}; -read_shebang_string(_) -> - error. - -node(ReadShebang, Scan, Inner0) -> - case ReadShebang(Inner0) of - {ok, Shebang, Inner, Loc} -> - continue(Scan, Inner, Loc, [Shebang]); - error -> - continue(Scan, Inner0, ?START_LOCATION, []) - end. +node(Scan, Inner0) -> + continue(Scan, Inner0, ?START_LOCATION, []). -spec continue(state()) -> node_ret(). continue(#state{scan = Scan, inner = Inner, loc = Loc, buffer = Buffer}) -> @@ -117,6 +90,17 @@ continue(#state{scan = Scan, inner = Inner, loc = Loc, buffer = Buffer}) -> continue(Scan, Inner0, Loc0, []) -> case Scan(Inner0, Loc0) of + {{ok, [{'#', _}, {'!', _} | _] = Tokens0, Loc}, Inner} -> + {Shebang, Buffer} = split_shebang(Tokens0), + Anno = [{text, Shebang}, {location, Loc0}], + State = #state{ + scan = Scan, + inner = Inner, + loc = Loc, + original = [], + buffer = Buffer + }, + {ok, [{shebang, Anno, Shebang}], [], State}; {{ok, Tokens, Loc}, Inner} -> continue(Scan, Inner, Loc, Tokens); {{error, Reason}, _Inner} -> @@ -136,12 +120,12 @@ continue(Scan, Inner0, Loc0, Buffer0) -> buffer = Buffer }, {ok, Tokens, Comments, State}; - {{eof, Loc}, _Inner} -> + {Eof, _Inner} when Eof =:= eof; element(1, Eof) =:= eof -> {Tokens, NodeTokens, Comments, []} = split_tokens(Buffer0, []), State = #state{ scan = fun eof/2, inner = undefined, - loc = Loc, + loc = Loc0, original = NodeTokens, buffer = [] }, @@ -171,6 +155,17 @@ eof(undefined, Loc) -> last_node_string(#state{original = Tokens}) -> stringify_tokens(Tokens). +split_shebang(Tokens) -> + {ShebangTokens, Rest} = lists:splitwith( + fun + ({white_space, _, [$\n | _]}) -> false; + (_) -> true + end, + Tokens + ), + {Shebang, _Anno} = stringify_tokens(ShebangTokens), + {unicode:characters_to_list(Shebang), Rest}. + %% TODO: make smarter stringify_tokens([Token | _] = Tokens) -> Anno = element(2, Token), From 56645df9930ab2443c613e061238b4fc17461181 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= Date: Wed, 8 Jul 2020 12:01:56 +0100 Subject: [PATCH 2/3] First pass on stdin reading --- rebar.config | 2 +- src/erlfmt.erl | 5 ++++- src/erlfmt_cli.erl | 25 +++++++++++++++++-------- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/rebar.config b/rebar.config index 9c481424..9b45df93 100644 --- a/rebar.config +++ b/rebar.config @@ -8,7 +8,7 @@ {escript_name, erlfmt}. -{escript_emu_args, "%%! +sbtu +A0 -noinput -mode minimal\n"}. +{escript_emu_args, "%%! +sbtu +A0 -noinput -noshell -mode minimal\n"}. {plugins, [erlfmt]}. diff --git a/src/erlfmt.erl b/src/erlfmt.erl index b9f6d210..e89a9c92 100644 --- a/src/erlfmt.erl +++ b/src/erlfmt.erl @@ -62,10 +62,11 @@ init(State) -> rebar3_fmt_prv:init(State). %% API entry point --spec format_file(file:name_all(), config()) -> +-spec format_file(file:name_all() | stdin, config()) -> {ok, [error_info()]} | {error, error_info()}. format_file(FileName, {Pragma, Out}) -> try + %% TODO: fix for stdin ShouldFormat = (Pragma == ignore) orelse contains_pragma_file(FileName), case ShouldFormat of true -> @@ -145,6 +146,8 @@ file_read_nodes(FileName) -> read_nodes(erlfmt_scan:io_node(File), FileName, [], []) end). +read_file(stdin, Action) -> + Action(standard_io); read_file(FileName, Action) -> case file:open(FileName, [read, {encoding, utf8}]) of {ok, File} -> diff --git a/src/erlfmt_cli.erl b/src/erlfmt_cli.erl index 8f1e5ed4..d2f55af1 100644 --- a/src/erlfmt_cli.erl +++ b/src/erlfmt_cli.erl @@ -32,21 +32,21 @@ opts() -> {require_pragma, undefined, "require-pragma", undefined, "Require a special comment @format, called a pragma, " "to be present in the file's first docblock comment in order for prettier to format it."}, - {files, undefined, undefined, string, "files to format"} + {files, undefined, undefined, string, "files to format, - for stdin"} ]. -spec do(list(), string()) -> ok. do(Opts, Name) -> case parse_opts(Opts, Name, [], #config{}) of - {format, [], _Config} -> - io:put_chars(standard_error, "no files to format provided\n\n"), - getopt:usage(opts(), Name), - erlang:halt(1); {format, Files, Config} -> - case format_files(Files, Config, false) of + case format_files(Files, Config, _HadErrors = false) of true -> erlang:halt(4); false -> ok - end + end; + {error, Message} -> + io:put_chars(standard_error, [Message, "\n\n"]), + getopt:usage(opts(), Name), + erlang:halt(1) end. format_files([FileName | FileNames], Config, HadErrors) -> @@ -88,9 +88,18 @@ parse_opts([require_pragma | Rest], Name, Files, Config) -> parse_opts(Rest, Name, Files, Config#config{require_pragma = true}); parse_opts([{files, NewFiles} | Rest], Name, Files0, Config) -> parse_opts(Rest, Name, expand_files(NewFiles, Files0), Config); +parse_opts([], _Name, [stdin], Config) -> + {format, [stdin], Config}; +parse_opts([], _Name, [], _Config) -> + {error, "no files provided to format"}; parse_opts([], _Name, Files, Config) -> - {format, lists:reverse(Files), Config}. + case lists:member(stdin, Files) of + true -> {error, "stdin mode can't be combined with other files"}; + false -> {format, lists:reverse(Files), Config} + end. +expand_files("-", Files) -> + [stdin | Files]; expand_files(NewFile, Files) when is_integer(hd(NewFile)) -> case filelib:is_regular(NewFile) of true -> From 7536bdc80dfccb925894f2773643478de0ecd3fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Muska=C5=82a?= Date: Wed, 8 Jul 2020 13:53:41 +0100 Subject: [PATCH 3/3] Finish stdio input & add tests --- src/erlfmt.erl | 42 ++++++++++++++++++------------ src/erlfmt_cli.erl | 7 +++++ test/erlfmt_SUITE.erl | 43 ++++++++++++++++++++++++++++--- test/erlfmt_SUITE_data/pragma.erl | 3 +++ 4 files changed, 75 insertions(+), 20 deletions(-) create mode 100644 test/erlfmt_SUITE_data/pragma.erl diff --git a/src/erlfmt.erl b/src/erlfmt.erl index e89a9c92..0a94a825 100644 --- a/src/erlfmt.erl +++ b/src/erlfmt.erl @@ -63,20 +63,17 @@ init(State) -> %% API entry point -spec format_file(file:name_all() | stdin, config()) -> - {ok, [error_info()]} | {error, error_info()}. + {ok, [error_info()]} | skip | {error, error_info()}. format_file(FileName, {Pragma, Out}) -> try - %% TODO: fix for stdin - ShouldFormat = (Pragma == ignore) orelse contains_pragma_file(FileName), - case ShouldFormat of - true -> - {ok, Nodes, Warnings} = file_read_nodes(FileName), + case file_read_nodes(FileName, Pragma) of + {ok, Nodes, Warnings} -> [$\n | Formatted] = format_nodes(Nodes), verify_nodes(FileName, Nodes, Formatted), write_formatted(FileName, Formatted, Out), {ok, Warnings}; - false -> - {ok, []} + skip -> + skip end catch {error, Error} -> {error, Error} @@ -119,7 +116,7 @@ contains_pragma_comment(_) -> {options, [{erlfmt_scan:location(), erlfmt_scan:location()}]}. format_range(FileName, StartLocation, EndLocation) -> try - {ok, Nodes, Warnings} = file_read_nodes(FileName), + {ok, Nodes, Warnings} = file_read_nodes(FileName, ignore), case verify_ranges(Nodes, StartLocation, EndLocation) of {ok, NodesInRange} -> [$\n | Result] = format_nodes(NodesInRange), @@ -136,14 +133,14 @@ format_range(FileName, StartLocation, EndLocation) -> -spec read_nodes(file:name_all()) -> {ok, [erlfmt_parse:abstract_form()], [error_info()]} | {error, error_info()}. read_nodes(FileName) -> - try file_read_nodes(FileName) + try file_read_nodes(FileName, ignore) catch {error, Error} -> {error, Error} end. -file_read_nodes(FileName) -> +file_read_nodes(FileName, Pragma) -> read_file(FileName, fun (File) -> - read_nodes(erlfmt_scan:io_node(File), FileName, [], []) + read_nodes(erlfmt_scan:io_node(File), FileName, Pragma) end). read_file(stdin, Action) -> @@ -162,17 +159,28 @@ read_file(FileName, Action) -> -spec read_nodes_string(file:name_all(), string()) -> {ok, [erlfmt_parse:abstract_form()], [error_info()]} | {error, error_info()}. read_nodes_string(FileName, String) -> - try read_nodes(erlfmt_scan:string_node(String), FileName, [], []) + try read_nodes(erlfmt_scan:string_node(String), FileName, ignore) catch {error, Error} -> {error, Error} end. -read_nodes({ok, Tokens, Comments, Cont}, FileName, Acc, Warnings0) -> +read_nodes({ok, Tokens, Comments, Cont}, FileName, Pragma) -> + {Node, Warnings} = parse_nodes(Tokens, Comments, FileName, Cont, []), + case contains_pragma_node(Node) of + false when Pragma =:= require -> + skip; + _ -> + read_nodes_loop(erlfmt_scan:continue(Cont), FileName, [Node], Warnings) + end; +read_nodes(Other, FileName, _Pragma) -> + read_nodes_loop(Other, FileName, [], []). + +read_nodes_loop({ok, Tokens, Comments, Cont}, FileName, Acc, Warnings0) -> {Node, Warnings} = parse_nodes(Tokens, Comments, FileName, Cont, Warnings0), - read_nodes(erlfmt_scan:continue(Cont), FileName, [Node | Acc], Warnings); -read_nodes({eof, _Loc}, _FileName, Acc, Warnings) -> + read_nodes_loop(erlfmt_scan:continue(Cont), FileName, [Node | Acc], Warnings); +read_nodes_loop({eof, _Loc}, _FileName, Acc, Warnings) -> {ok, lists:reverse(Acc), lists:reverse(Warnings)}; -read_nodes({error, {ErrLoc, Mod, Reason}, _Loc}, FileName, _Acc, _Warnings) -> +read_nodes_loop({error, {ErrLoc, Mod, Reason}, _Loc}, FileName, _Acc, _Warnings) -> throw({error, {FileName, ErrLoc, Mod, Reason}}). parse_nodes([], _Comments, _FileName, Cont, Warnings) -> diff --git a/src/erlfmt_cli.erl b/src/erlfmt_cli.erl index d2f55af1..41263fe8 100644 --- a/src/erlfmt_cli.erl +++ b/src/erlfmt_cli.erl @@ -64,6 +64,11 @@ format_files([FileName | FileNames], Config, HadErrors) -> {ok, Warnings} -> [print_error_info(Warning) || Warning <- Warnings], format_files(FileNames, Config, HadErrors); + skip when Config#config.verbose -> + io:format(standard_error, "Skipping ~s because of missing @format pragma\n", [FileName]), + format_files(FileNames, Config, HadErrors); + skip -> + format_files(FileNames, Config, HadErrors); {error, Error} -> print_error_info(Error), format_files(FileNames, Config, true) @@ -88,6 +93,8 @@ parse_opts([require_pragma | Rest], Name, Files, Config) -> parse_opts(Rest, Name, Files, Config#config{require_pragma = true}); parse_opts([{files, NewFiles} | Rest], Name, Files0, Config) -> parse_opts(Rest, Name, expand_files(NewFiles, Files0), Config); +parse_opts([], _Name, [stdin], #config{out = Out}) when Out =/= standard_out -> + {error, "stdin mode can't be combined with out or write options"}; parse_opts([], _Name, [stdin], Config) -> {format, [stdin], Config}; parse_opts([], _Name, [], _Config) -> diff --git a/test/erlfmt_SUITE.erl b/test/erlfmt_SUITE.erl index d8f0d5d2..28629b97 100644 --- a/test/erlfmt_SUITE.erl +++ b/test/erlfmt_SUITE.erl @@ -48,6 +48,10 @@ annos/1, shebang/1, smoke_test_cli/1, + smoke_test_stdio_escript/1, + smoke_test_stdio_regular/1, + smoke_test_stdio_without_pragma/1, + smoke_test_stdio_with_pragma/1, snapshot_simple_comments/1, snapshot_big_binary/1, snapshot_attributes/1, @@ -104,7 +108,11 @@ groups() -> ]}, {smoke_tests, [parallel], [ {group, snapshot_tests}, - smoke_test_cli + smoke_test_cli, + smoke_test_stdio_escript, + smoke_test_stdio_regular, + smoke_test_stdio_without_pragma, + smoke_test_stdio_with_pragma ]}, {snapshot_tests, [parallel], [ snapshot_simple_comments, @@ -918,9 +926,38 @@ parse_forms(String) -> end. smoke_test_cli(Config) when is_list(Config) -> + ?assertMatch("Usage: erlfmt " ++ _, os:cmd(escript() ++ " -h")). + +smoke_test_stdio_escript(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + Path = filename:join(DataDir, "escript.erl"), + Formatted = os:cmd("cat " ++ Path ++ " | " ++ escript() ++ " -"), + {ok, Expected} = file:read_file(Path), + ?assertEqual(Expected, unicode:characters_to_binary(Formatted)). + +smoke_test_stdio_regular(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + Path = filename:join(DataDir, "attributes.erl"), + Formatted = os:cmd("cat " ++ Path ++ " | " ++ escript() ++ " -"), + {ok, Expected} = file:read_file(Path), + ?assertEqual(Expected, unicode:characters_to_binary(Formatted)). + +smoke_test_stdio_without_pragma(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + Path = filename:join(DataDir, "big_binary.erl"), + Formatted = os:cmd("cat " ++ Path ++ " | " ++ escript() ++ " - --require-pragma"), + ?assertEqual("", Formatted). + +smoke_test_stdio_with_pragma(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + Path = filename:join(DataDir, "pragma.erl"), + Formatted= os:cmd("cat " ++ Path ++ " | " ++ escript() ++ " - --require-pragma"), + {ok, Expected} = file:read_file(Path), + ?assertEqual(Expected, unicode:characters_to_binary(Formatted)). + +escript() -> %% this relies on the _build structure rebar3 uses - Escript = filename:join(code:lib_dir(erlfmt), "../../bin/erlfmt"), - ?assertMatch("Usage: erlfmt " ++ _, os:cmd(Escript ++ " -h")). + filename:join(code:lib_dir(erlfmt), "../../bin/erlfmt"). snapshot_simple_comments(Config) -> snapshot_same("simple_comments.erl", Config). diff --git a/test/erlfmt_SUITE_data/pragma.erl b/test/erlfmt_SUITE_data/pragma.erl new file mode 100644 index 00000000..842bbeea --- /dev/null +++ b/test/erlfmt_SUITE_data/pragma.erl @@ -0,0 +1,3 @@ +%% @format + +-module(pragma).