From 855373ddf8f36bdd2168a4eba4fec0248682a7dc Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Fri, 29 Mar 2019 21:13:53 -0700 Subject: [PATCH 1/3] Implement `for f.(x)` syntax --- base/show.jl | 4 +++ doc/src/manual/arrays.md | 14 +++++++++ src/julia-parser.scm | 62 ++++++++++++++++++++++++++++++--------- src/julia-syntax.scm | 22 ++++++++++---- test/broadcast.jl | 8 +++++ test/syntax.jl | 63 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 154 insertions(+), 19 deletions(-) diff --git a/base/show.jl b/base/show.jl index 1d2087ced8e18..4140c87f34833 100644 --- a/base/show.jl +++ b/base/show.jl @@ -1341,6 +1341,10 @@ function show_unquoted(io::IO, ex::Expr, indent::Int, prec::Int) print(io, head, ' ') show_list(io, args, ", ", indent) + elseif nargs == 1 && head == :fordot + print(io, "for ") + show_list(io, args, ", ", indent) + elseif head === :macrocall && nargs >= 2 # first show the line number argument as a comment if isa(args[2], LineNumberNode) || is_expr(args[2], :line) diff --git a/doc/src/manual/arrays.md b/doc/src/manual/arrays.md index 4e8b4d4109486..c45827399a05c 100644 --- a/doc/src/manual/arrays.md +++ b/doc/src/manual/arrays.md @@ -889,6 +889,20 @@ julia> string.(1:3, ". ", ["First", "Second", "Third"]) "3. Third" ``` +Normal dot calls are evaluated (or _materialized_) immediately. To construct an +intermediate representation without invoking the computation, prepend `for` to +the a dot calls. + +```jldoctest +julia> bc = for (1:3).^2; + +julia> bc isa Broadcast.Broadcasted # it's not an `Array` +true + +julia> sum(bc) # summation without allocating any arrays +14 +``` + ## Implementation The base array type in Julia is the abstract type [`AbstractArray{T,N}`](@ref). It is parameterized by diff --git a/src/julia-parser.scm b/src/julia-parser.scm index e139361ef6fea..7a3f299d18af4 100644 --- a/src/julia-parser.scm +++ b/src/julia-parser.scm @@ -1330,11 +1330,18 @@ ((while) (begin0 (list 'while (parse-cond s) (parse-block s)) (expect-end s word))) ((for) - (let* ((ranges (parse-comma-separated-iters s)) - (body (parse-block s))) - (expect-end s word) - `(for ,(if (length= ranges 1) (car ranges) (cons 'block ranges)) - ,body))) + (let ((r (parse-iteration-spec-or-dotcall s))) + (case (car r) + ;; for loop + ((for) + (let* ((ranges (parse-comma-separated-iters-continued s (list (cdr r)))) + (body (parse-block s))) + (expect-end s word) + `(for ,(if (length= ranges 1) (car ranges) (cons 'block ranges)) + ,body))) + ;; lazy dotcall + (else + `(fordot ,(cdr r)))))) ((let) (let ((binds (if (memv (peek-token s) '(#\newline #\;)) @@ -1622,6 +1629,12 @@ ;; as above, but allows both "i=r" and "i in r" (define (parse-iteration-spec s) + (let ((r (parse-iteration-spec-or-dotcall s))) + (case (car r) + ((for) (cdr r)) + (else (error "invalid iteration specification"))))) + +(define (parse-iteration-spec-or-dotcall s) (let* ((outer? (if (eq? (peek-token s) 'outer) (begin (take-token s) @@ -1643,19 +1656,40 @@ ;; should be: (error "invalid iteration specification") (parser-depwarn s (string "for " (deparse `(= ,lhs ,rhs)) " " t) (string "for " (deparse `(= ,lhs ,rhs)) "; " t))) - (if outer? + (cons + 'for + (if outer? `(= (outer ,lhs) ,rhs) - `(= ,lhs ,rhs)))) + `(= ,lhs ,rhs))))) ((and (eq? lhs ':) (closing-token? t)) - ':) - (else (error "invalid iteration specification"))))) + '(for . :)) + + (else + (if (dotcall? lhs) + (cons 'fordot lhs) + (error "invalid expression after `for`")))))) + +(define (dotcall? ex) + (and (pair? ex) + (case (car ex) + ((call) + (equal? (substring (string (cadr ex)) 0 1) ".")) + ((|.|) (and (pair? (cdr ex)) + (pair? (cddr ex)) + (pair? (caddr ex)) + (eq? (caaddr ex) 'tuple))) + (else #f)))) (define (parse-comma-separated-iters s) - (let loop ((ranges '())) - (let ((r (parse-iteration-spec s))) - (case (peek-token s) - ((#\,) (take-token s) (loop (cons r ranges))) - (else (reverse! (cons r ranges))))))) + (parse-comma-separated-iters-continued s (list (parse-iteration-spec s)))) + +(define (parse-comma-separated-iters-continued s ranges) + (case (peek-token s) + ((#\,) + (take-token s) + (parse-comma-separated-iters-continued s (cons (parse-iteration-spec s) ranges))) + (else + (reverse! ranges)))) (define (parse-space-separated-exprs s) (with-space-sensitive diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index dbd47925d68ff..a01262a7f0d31 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1642,10 +1642,7 @@ `(block ,@stmts ,nuref)) expr)) -; lazily fuse nested calls to expr == f.(args...) into a single broadcast call, -; or a broadcast! call if lhs is non-null. -(define (expand-fuse-broadcast lhs rhs) - (define (fuse? e) (and (pair? e) (eq? (car e) 'fuse))) +(define (expand-lazy-broadcast rhs) (define (dot-to-fuse e (top #f)) ; convert e == (. f (tuple args)) to (fuse f args) (define (make-fuse f args) ; check for nested (fuse f args) exprs and combine (define (split-kwargs args) ; return (cons keyword-args positional-args) extracted from args @@ -1682,7 +1679,15 @@ (list '^ (car x) (expand-forms `(call (call (core apply_type) (top Val) ,(cadr x)))))) (make-fuse f x))) e))) - (let ((e (dot-to-fuse rhs #t)) ; an expression '(fuse func args) if expr is a dot call + (dot-to-fuse rhs #t)) + +(define (fuse? e) (and (pair? e) (eq? (car e) 'fuse))) + +; lazily fuse nested calls to expr == f.(args...) into a single broadcast call, +; or a broadcast! call if lhs is non-null. +(define (expand-fuse-broadcast lhs rhs) + (let ((e ; an expression '(fuse func args) if expr is a dot call + (expand-lazy-broadcast rhs)) (lhs-view (ref-to-view lhs))) ; x[...] expressions on lhs turn in to view(x, ...) to update x in-place (if (fuse? e) ; expanded to a fuse op call @@ -1839,6 +1844,13 @@ (lambda (e) (expand-fuse-broadcast (cadr e) (caddr e))) + 'fordot + (lambda (e) + (let ((x (expand-lazy-broadcast (cadr e)))) + (if (fuse? x) + (expand-forms (cdr x)) + (error "non-dot call after `for`")))) + '|<:| (lambda (e) (expand-forms `(call |<:| ,@(cdr e)))) '|>:| diff --git a/test/broadcast.jl b/test/broadcast.jl index 3e39d14cfb440..d181be91a3f31 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -182,6 +182,14 @@ let x = [1, 3.2, 4.7], @test atan.(α, y') == broadcast(atan, α, y') end +@testset "for f.(args...) syntax (#19198, #31088)" begin + let bc = for (1:3).^2 + @test bc isa Broadcast.Broadcasted + @test sum(bc) == 14 + end + @test sum(for (1:3).^2) == 14 +end + # issue 14725 let a = Number[2, 2.0, 4//2, 2+0im] / 2 @test eltype(a) == Number diff --git a/test/syntax.jl b/test/syntax.jl index 1626b57b1085f..40e3d8b5fd689 100644 --- a/test/syntax.jl +++ b/test/syntax.jl @@ -1839,3 +1839,66 @@ end # issue #31404 f31404(a, b; kws...) = (a, b, kws.data) @test f31404(+, (Type{T} where T,); optimize=false) === (+, (Type,), (optimize=false,)) + +# lazy dot call +@test Meta.parse("for f.(x)") == Expr(:fordot, :(f.(x))) +@test Meta.parse("for f.(g(x))") == Expr(:fordot, :(f.(g(x)))) +@test Meta.parse("for .+ x") == Expr(:fordot, :(.+ x)) +@test Meta.parse("for .+ g(x)") == Expr(:fordot, :(.+ g(x))) + +@test Meta.parse(""" +for x in for f.(xs) +end +""") == Expr(:for, + Expr(:(=), :x, Expr(:fordot, :(f.(xs)))), + Expr(:block, LineNumberNode(2, :none))) + +@test Meta.parse(""" +for x in for f.(xs), y in for g.(ys) +end +""") == Expr(:for, + Expr(:block, + Expr(:(=), :x, Expr(:fordot, :(f.(xs)))), + Expr(:(=), :y, Expr(:fordot, :(g.(ys))))), + Expr(:block, LineNumberNode(2, :none))) + +@testset for valid in [ + "for f.(x)" + "for f.(g(x))" + "for .+ x" + "for .+ g(x)" + "for f.(args[1], args[2])" + "for f.(args...)" + ] + @test Meta.lower(@__MODULE__, Meta.parse(valid)).head == :thunk +end + +@testset for invalid in [ + "for x" + "for f(x)" + "for f(g.(x))" + "for + x" + "for + f.(x))" + ] + @test_throws ParseError Meta.parse(invalid) +end + +@testset for code in [ + "for f.(x)" + "for f.(g(x))" + "for (.+)(x)" + "for (.+)(g(x))" + ] + @test string(Meta.parse(code)) == code +end + +@testset for nondot in [ + :(f(x)) + :(f(g.(x))) + :(+f.(x)) + 1 + quote end + ] + @test Meta.lower(@__MODULE__, Expr(:fordot, nondot)) == + Expr(:error, "non-dot call after `for`") +end From a23e47b9458cbb5ea23b3e61f00caf1321ec0e76 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Fri, 29 Mar 2019 21:14:00 -0700 Subject: [PATCH 2/3] Add NEWS.md --- NEWS.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS.md b/NEWS.md index 012e6747ec3a5..2d69565e648e0 100644 --- a/NEWS.md +++ b/NEWS.md @@ -14,6 +14,9 @@ New language features * `inv(::Missing)` has now been added and returns `missing` ([#31408]). + * A new syntax `for f.(...)` is added to create a non-`materialize`d `Broadcasted` object + from a given "dot call" expression ([#19198]). + Multi-threading changes ----------------------- From 1882eae33605b4748f2c86bac18be0132cbf9a77 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Fri, 29 Mar 2019 21:59:58 -0700 Subject: [PATCH 3/3] Document new surface syntax in devdocs/ast.md --- doc/src/devdocs/ast.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/src/devdocs/ast.md b/doc/src/devdocs/ast.md index ef8860e375895..652798cbc94b4 100644 --- a/doc/src/devdocs/ast.md +++ b/doc/src/devdocs/ast.md @@ -26,6 +26,8 @@ For example `(call f x)` corresponds to `Expr(:call, :f, :x)` in Julia. | `f(x, y=1, z=2)` | `(call f x (kw y 1) (kw z 2))` | | `f(x; y=1)` | `(call f (parameters (kw y 1)) x)` | | `f(x...)` | `(call f (... x))` | +| `f.(x)` | `(. f (tuple x))` | +| `for f.(x)` | `(fordot (. f (tuple x)))` | `do` syntax: @@ -48,6 +50,7 @@ call. Finally, chains of comparisons have their own special expression structure | Input | AST | |:----------- |:------------------------- | | `x+y` | `(call + x y)` | +| `x.+y` | `(call .+ x y)` | | `a+b+c+d` | `(call + a b c d)` | | `2x` | `(call * 2 x)` | | `a&&b` | `(&& a b)` | @@ -59,7 +62,6 @@ call. Finally, chains of comparisons have their own special expression structure | `a==b` | `(call == a b)` | | `1