Skip to content

Commit

Permalink
Merge pull request #46 from akash-akya/dev
Browse files Browse the repository at this point in the history 
Add support for redirecting stderr to stdout
  • Loading branch information
@akash-akya
akash-akya authored Jul 8, 2024
2 parents cea756c + 6e8cc29 commit 905b4df
Show file tree
Hide file tree
Showing 8 changed files with 182 additions and 43 deletions.
12 changes: 11 additions & 1 deletion c_src/spawner.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,9 +146,19 @@ static int exec_process(char const *bin, char *const *args, int socket,
_exit(FORK_EXEC_FAILURE);
}

if (strcmp(stderr_str, "consume") == 0) {
if (strcmp(stderr_str, "redirect_to_stdout") == 0) {
close(STDERR_FILENO);
close(r_cmderr);
close(w_cmderr);

if (dup2(w_cmdout, STDERR_FILENO) < 0) {
perror("[spawner] failed to redirect stderr to stdout");
_exit(FORK_EXEC_FAILURE);
}
} else if (strcmp(stderr_str, "consume") == 0) {
close(STDERR_FILENO);
close(r_cmderr);

if (dup2(w_cmderr, STDERR_FILENO) < 0) {
perror("[spawner] failed to dup to stderr");
_exit(FORK_EXEC_FAILURE);
Expand Down
33 changes: 26 additions & 7 deletions lib/exile.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,18 +119,26 @@ defmodule Exile do
"X 250 X\n"
```
With stderr set to :redirect_to_stdout
```
iex> Exile.stream!(["sh", "-c", "echo foo; echo bar >> /dev/stderr"], stderr: :redirect_to_stdout)
...> |> Enum.into("")
"foo\nbar\n"
```
With stderr set to :consume
```
iex> Exile.stream!(["sh", "-c", "echo foo\necho bar >> /dev/stderr"], stderr: :consume)
iex> Exile.stream!(["sh", "-c", "echo foo; echo bar >> /dev/stderr"], stderr: :consume)
...> |> Enum.to_list()
[{:stdout, "foo\n"}, {:stderr, "bar\n"}]
```
With stderr set to :disable
```
iex> Exile.stream!(["sh", "-c", "echo foo\necho bar >> /dev/stderr"], stderr: :disable)
iex> Exile.stream!(["sh", "-c", "echo foo; echo bar >> /dev/stderr"], stderr: :disable)
...> |> Enum.to_list()
["foo\n"]
```
Expand Down Expand Up @@ -195,13 +203,16 @@ defmodule Exile do
Chunk size can be less than the `max_chunk_size` depending on the amount of
data available to be read. Defaults to `65_535`
* `stderr` - different ways to handle stderr stream. possible values `:console`, `:disable`, `:stream`.
* `stderr` - different ways to handle stderr stream.
1. `:console` - stderr output is redirected to console (Default)
2. `:disable` - stderr output is redirected `/dev/null` suppressing all output
3. `:consume` - connects stderr for the consumption. The output stream will contain stderr
2. `:redirect_to_stdout` - stderr output is redirected to stdout
3. `:disable` - stderr output is redirected `/dev/null` suppressing all output
4. `:consume` - connects stderr for the consumption. The output stream will contain stderr
data along with stdout. Stream data will be either `{:stdout, iodata}` or `{:stderr, iodata}`
to differentiate different streams. See example below.
See [`:stderr`](`m:Exile.Process#module-stderr`) for more details and issues associated with them.
* `ignore_epipe` - When set to true, reader can exit early without raising error.
Typically writer gets `EPIPE` error on write when program terminate prematurely.
With `ignore_epipe` set to true this error will be ignored. This can be used to
Expand All @@ -221,6 +232,14 @@ defmodule Exile do
|> Stream.run()
```
Stream with stderr redirected to stdout
```
Exile.stream!(["sh", "-c", "echo foo; echo bar >> /dev/stderr"], stderr: :redirect_to_stdout)
|> Stream.map(&IO.write/1)
|> Stream.run()
```
Stream with stderr
```
Expand Down Expand Up @@ -257,7 +276,7 @@ defmodule Exile do
@spec stream!(nonempty_list(String.t()),
input: Enum.t() | collectable_func(),
exit_timeout: timeout(),
stderr: :console | :disable | :consume,
stderr: :console | :redirect_to_stdout | :disable | :consume,
ignore_epipe: boolean(),
max_chunk_size: pos_integer()
) :: Exile.Stream.t()
Expand All @@ -278,7 +297,7 @@ defmodule Exile do
@spec stream(nonempty_list(String.t()),
input: Enum.t() | collectable_func(),
exit_timeout: timeout(),
stderr: :console | :disable | :consume,
stderr: :console | :redirect_to_stdout | :disable | :consume,
ignore_epipe: boolean(),
max_chunk_size: pos_integer()
) :: Exile.Stream.t()
Expand Down
105 changes: 83 additions & 22 deletions lib/exile/process.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,25 +104,84 @@ defmodule Exile.Process do
### Pipe Operations
Pipe owner can read or write date to the owned pipe. `:stderr` by
default is connected to console, data written to stderr will appear on
the console. You can enable reading stderr by passing `stderr: :consume`
during process creation.
Special function `Exile.Process.read_any/2` can be used to read
from either stdout or stderr whichever has the data available.
All Pipe operations blocks the caller to have blocking as natural
back-pressure and to make the API simple. This is an important
feature of Exile, that is the ability to block caller when the stdio
buffer is full, exactly similar to how programs works on the shell
with pipes between then `cat larg-file | grep "foo"`. Internally it
does not block the Exile process or VM (which is typically the case
with NIF calls). Because of this user can make concurrent read,
write to different pipes from separate processes. Internally Exile
uses asynchronous IO APIs to avoid blocking of VM or VM process.
Reading from stderr
Only Pipe owner can read or write date to the owned pipe.
All Pipe operations (read/write) blocks the caller as a mechanism
to put back-pressure, and this also makes the API simpler.
This is same as how command-line programs works on the shell,
along with pipes in-between, Example: `cat larg-file | grep "foo"`.
Internally Exile uses asynchronous IO APIs to avoid blocking VM
(by default NIF calls blocks the VM scheduler),
so you can open several pipes and do concurrent IO operations without
blocking VM.
### `stderr`
by default is `:stderr` is connected to console, data written to
stderr will appear on the console.
You can change the behavior by setting `:stderr`:
1. `:console` - stderr output is redirected to console (Default)
2. `:redirect_to_stdout` - stderr output is redirected to stdout
2. `:consume` - stderr output read separately, allowing you to consume it separately from stdout. See below for more details
4. `:disable` - stderr output is redirected `/dev/null` suppressing all output. See below for more details.
### Using `redirect_to_stdout`
stderr data will be redirected to stdout. When you read stdout
you will see both stdout & stderr combined and you won't be
able differentiate stdout and stderr separately.
This is similar to `:stderr_to_stdout` option present in
[Ports](https://www.erlang.org/doc/apps/erts/erlang.html#open_port/2).
> #### Unexpected Behaviors {: .warning}
>
> On many systems, `stdout` and `stderr` are separated. And between
> the source program to Exile, via the kernel, there are several places
> that may buffer data, even temporarily, before Exile is ready
> to read them. There is no enforced ordering of the readiness of
> these independent buffers for Exile to make use of.
>
> This can result in unexpected behavior, including:
>
> * mangled data, for example, UTF-8 characters may be incomplete
> until an additional buffered segment is released on the same
> source
> * raw data, where binary data sent on one source, is incompatible
> with data sent on the other source.
> * interleaved data, where what appears to be synchronous, is not
>
> In short, the two streams might be combined at arbitrary byte position
> leading to above mentioned issue.
>
> Most well-behaved command-line programs are unlikely to exhibit
> this, but you need to be aware of the risk.
>
> A good example of this unexpected behavior is streaming JSON from
> an external tool to Exile, where normal JSON output is expected on
> stdout, and errors or warnings via stderr. In the case of an
> unexpected error, the stdout stream could be incomplete, or the
> stderr message might arrive before the closing data on the stdout
> stream.
### Using `consume`
stderr data can be consumed separately using
`Exile.Process.read_stderr/2`. Special function
`Exile.Process.read_any/2` can be used to read from either stdout or
stderr whichever has the data available. See the examples for more
details.
> #### Unexpected Behaviors {: .warning}
>
> When set, the `stderr` output **MUST** be consumed to
> avoid blocking the external program when stderr buffer is full.
Reading from stderr using `read_stderr`
```
# write "Hello" to stdout and "World" to stderr
Expand Down Expand Up @@ -301,12 +360,14 @@ defmodule Exile.Process do
These can be accessed in the external program
* `stderr` - different ways to handle stderr stream.
possible values `:console`, `:disable`, `:stream`.
1. `:console` - stderr output is redirected to console (Default)
2. `:disable` - stderr output is redirected `/dev/null` suppressing all output
3. `:consume` - connects stderr for the consumption. When set to stream the output must be consumed to
2. `:redirect_to_stdout` - stderr output is redirected to stdout
3. `:disable` - stderr output is redirected `/dev/null` suppressing all output
4. `:consume` - connects stderr for the consumption. When set, the stderr output must be consumed to
avoid external program from blocking.
See [`:stderr`](#module-stderr) for more details and issues associated with them
Caller of the process will be the owner owner of the Exile Process.
And default owner of all opened pipes.
Expand Down
11 changes: 6 additions & 5 deletions lib/exile/process/exec.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ defmodule Exile.Process.Exec do
cmd_with_args: nonempty_list(),
cd: charlist,
env: env,
stderr: :console | :disable | :consume
stderr: :console | :redirect_to_stdout | :disable | :consume
}}
| {:error, String.t()}
def normalize_exec_args(cmd_with_args, opts) do
Expand Down Expand Up @@ -192,18 +192,19 @@ defmodule Exile.Process.Exec do
end
end

@spec normalize_stderr(stderr :: :console | :disable | :consume | nil) ::
{:ok, :console | :disable | :consume} | {:error, String.t()}
@spec normalize_stderr(stderr :: :console | :redirect_to_stdout | :disable | :consume | nil) ::
{:ok, :console | :redirect_to_stdout | :disable | :consume} | {:error, String.t()}
defp normalize_stderr(stderr) do
case stderr do
nil ->
{:ok, :console}

stderr when stderr in [:console, :disable, :consume] ->
stderr when stderr in [:redirect_to_stdout, :console, :disable, :consume] ->
{:ok, stderr}

_ ->
{:error, ":stderr must be an atom and one of :console, :disable, :consume"}
{:error,
":stderr must be an atom and one of :redirect_to_stdout, :console, :disable, :consume"}
end
end

Expand Down
2 changes: 1 addition & 1 deletion lib/exile/process/state.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ defmodule Exile.Process.State do

@type read_mode :: :stdout | :stderr | :stdout_or_stderr

@type stderr_mode :: :console | :disable | :consume
@type stderr_mode :: :console | :redirect_to_stdout | :disable | :consume

@type pipes :: %{
stdin: Pipe.t(),
Expand Down
5 changes: 3 additions & 2 deletions lib/exile/stream.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -297,11 +297,12 @@ defmodule Exile.Stream do
nil ->
{:ok, :console}

stderr when stderr in [:console, :disable, :consume] ->
stderr when stderr in [:console, :redirect_to_stdout, :disable, :consume] ->
{:ok, stderr}

_ ->
{:error, ":stderr must be an atom and one of :console, :disable, :consume"}
{:error,
":stderr must be an atom and one of :console, :redirect_to_stdout, :disable, :consume"}
end
end

Expand Down
10 changes: 5 additions & 5 deletions mix.lock
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
%{
"bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"},
"credo": {:hex, :credo, "1.7.6", "b8f14011a5443f2839b04def0b252300842ce7388f3af177157c86da18dfbeea", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "146f347fb9f8cbc5f7e39e3f22f70acbef51d441baa6d10169dd604bfbc55296"},
"credo": {:hex, :credo, "1.7.7", "771445037228f763f9b2afd612b6aa2fd8e28432a95dbbc60d8e03ce71ba4446", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8bc87496c9aaacdc3f90f01b7b0582467b69b4bd2441fe8aae3109d843cc2f2e"},
"dialyxir": {:hex, :dialyxir, "1.4.3", "edd0124f358f0b9e95bfe53a9fcf806d615d8f838e2202a9f430d59566b6b53b", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "bf2cfb75cd5c5006bec30141b131663299c661a864ec7fbbc72dfa557487a986"},
"earmark_parser": {:hex, :earmark_parser, "1.4.39", "424642f8335b05bb9eb611aa1564c148a8ee35c9c8a8bba6e129d51a3e3c6769", [:mix], [], "hexpm", "06553a88d1f1846da9ef066b87b57c6f605552cfbe40d20bd8d59cc6bde41944"},
"elixir_make": {:hex, :elixir_make, "0.8.3", "d38d7ee1578d722d89b4d452a3e36bcfdc644c618f0d063b874661876e708683", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:certifi, "~> 2.0", [hex: :certifi, repo: "hexpm", optional: true]}], "hexpm", "5c99a18571a756d4af7a4d89ca75c28ac899e6103af6f223982f09ce44942cc9"},
"earmark_parser": {:hex, :earmark_parser, "1.4.40", "f3534689f6b58f48aa3a9ac850d4f05832654fe257bf0549c08cc290035f70d5", [:mix], [], "hexpm", "cdb34f35892a45325bad21735fadb88033bcb7c4c296a999bde769783f53e46a"},
"elixir_make": {:hex, :elixir_make, "0.8.4", "4960a03ce79081dee8fe119d80ad372c4e7badb84c493cc75983f9d3bc8bde0f", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:certifi, "~> 2.0", [hex: :certifi, repo: "hexpm", optional: true]}], "hexpm", "6e7f1d619b5f61dfabd0a20aa268e575572b542ac31723293a4c1a567d5ef040"},
"erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"},
"ex_doc": {:hex, :ex_doc, "0.33.0", "690562b153153c7e4d455dc21dab86e445f66ceba718defe64b0ef6f0bd83ba0", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "3f69adc28274cb51be37d09b03e4565232862a4b10288a3894587b0131412124"},
"ex_doc": {:hex, :ex_doc, "0.34.1", "9751a0419bc15bc7580c73fde506b17b07f6402a1e5243be9e0f05a68c723368", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "d441f1a86a235f59088978eff870de2e815e290e44a8bd976fe5d64470a4c9d2"},
"excoveralls": {:hex, :excoveralls, "0.18.1", "a6f547570c6b24ec13f122a5634833a063aec49218f6fff27de9df693a15588c", [:mix], [{:castore, "~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "d65f79db146bb20399f23046015974de0079668b9abb2f5aac074d078da60b8d"},
"file_system": {:hex, :file_system, "1.0.0", "b689cc7dcee665f774de94b5a832e578bd7963c8e637ef940cd44327db7de2cd", [:mix], [], "hexpm", "6752092d66aec5a10e662aefeed8ddb9531d79db0bc145bb8c40325ca1d8536d"},
"jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"},
"jason": {:hex, :jason, "1.4.3", "d3f984eeb96fe53b85d20e0b049f03e57d075b5acda3ac8d465c969a2536c17b", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "9a90e868927f7c777689baa16d86f4d0e086d968db5c05d917ccff6d443e58a3"},
"makeup": {:hex, :makeup, "1.1.2", "9ba8837913bdf757787e71c1581c21f9d2455f4dd04cfca785c70bbfff1a76a3", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cce1566b81fbcbd21eca8ffe808f33b221f9eee2cbc7a1706fc3da9ff18e6cac"},
"makeup_elixir": {:hex, :makeup_elixir, "0.16.2", "627e84b8e8bf22e60a2579dad15067c755531fea049ae26ef1020cad58fe9578", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "41193978704763f6bbe6cc2758b84909e62984c7752b3784bd3c218bb341706b"},
"makeup_erlang": {:hex, :makeup_erlang, "1.0.0", "6f0eff9c9c489f26b69b61440bf1b238d95badae49adac77973cbacae87e3c2e", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "ea7a9307de9d1548d2a72d299058d1fd2339e3d398560a0e46c27dab4891e4d2"},
Expand Down
47 changes: 47 additions & 0 deletions test/exile_test.exs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,53 @@ defmodule ExileTest do
assert IO.iodata_to_binary(stderr) == "Hello World\n"
end

test "stderr redirect_to_stdout" do
merged_output =
Exile.stream!(
[fixture("write_stderr.sh"), "Hello World"],
stderr: :redirect_to_stdout
)
|> Enum.to_list()
|> IO.iodata_to_binary()

assert merged_output == "Hello World\n"
end

test "order must be preserved when stderr is redirect to stdout" do
merged_output =
Exile.stream!(
["sh", "-c", "for s in $(seq 1 10); do echo stdout $s; echo stderr $s >&2; done"],
stderr: :redirect_to_stdout,
ignore_epipe: true
)
|> Enum.to_list()
|> IO.iodata_to_binary()
|> String.trim()

assert [
"stdout 1",
"stderr 1",
"stdout 2",
"stderr 2",
"stdout 3",
"stderr 3",
"stdout 4",
"stderr 4",
"stdout 5",
"stderr 5",
"stdout 6",
"stderr 6",
"stdout 7",
"stderr 7",
"stdout 8",
"stderr 8",
"stdout 9",
"stderr 9",
"stdout 10",
"stderr 10"
] == String.split(merged_output, "\n")
end

test "multiple streams" do
script = """
for i in {1..1000}; do
Expand Down

0 comments on commit 905b4df

Please sign in to comment.