Implement for f.(args...) syntax

NEWS.md

@@ -14,6 +14,9 @@ New language features
 
   * `inv(::Missing)` has now been added and returns `missing` ([#31408]).
 
+  * A new syntax `for f.(...)` is added to create a non-`materialize`d `Broadcasted` object
+    from a given "dot call" expression ([#19198]).
+
 Multi-threading changes
 -----------------------
 

base/show.jl

@@ -1341,6 +1341,10 @@ function show_unquoted(io::IO, ex::Expr, indent::Int, prec::Int)
         print(io, head, ' ')
         show_list(io, args, ", ", indent)
 
+    elseif nargs == 1 && head == :fordot
+        print(io, "for ")
+        show_list(io, args, ", ", indent)
+
     elseif head === :macrocall && nargs >= 2
         # first show the line number argument as a comment
         if isa(args[2], LineNumberNode) || is_expr(args[2], :line)

doc/src/devdocs/ast.md

@@ -26,6 +26,8 @@ For example `(call f x)` corresponds to `Expr(:call, :f, :x)` in Julia.
 | `f(x, y=1, z=2)` | `(call f x (kw y 1) (kw z 2))`     |
 | `f(x; y=1)`      | `(call f (parameters (kw y 1)) x)` |
 | `f(x...)`        | `(call f (... x))`                 |
+| `f.(x)`          | `(. f (tuple x))`                  |
+| `for f.(x)`      | `(fordot (. f (tuple x)))`         |
 
 `do` syntax:
 
@@ -48,6 +50,7 @@ call. Finally, chains of comparisons have their own special expression structure
 | Input       | AST                       |
 |:----------- |:------------------------- |
 | `x+y`       | `(call + x y)`            |
+| `x.+y`      | `(call .+ x y)`           |
 | `a+b+c+d`   | `(call + a b c d)`        |
 | `2x`        | `(call * 2 x)`            |
 | `a&&b`      | `(&& a b)`                |
@@ -59,7 +62,6 @@ call. Finally, chains of comparisons have their own special expression structure
 | `a==b`      | `(call == a b)`           |
 | `1<i<=n`    | `(comparison 1 < i <= n)` |
 | `a.b`       | `(. a (quote b))`         |
-| `a.(b)`     | `(. a b)`                 |
 
 ### Bracketed forms
 

doc/src/manual/arrays.md

@@ -889,6 +889,20 @@ julia> string.(1:3, ". ", ["First", "Second", "Third"])
  "3. Third"
 ```
 
+Normal dot calls are evaluated (or _materialized_) immediately.  To construct an
+intermediate representation without invoking the computation, prepend `for` to
+the a dot calls.
+
+```jldoctest
+julia> bc = for (1:3).^2;
+
+julia> bc isa Broadcast.Broadcasted  # it's not an `Array`
+true
+
+julia> sum(bc)  # summation without allocating any arrays
+14
+```
+
 ## Implementation
 
 The base array type in Julia is the abstract type [`AbstractArray{T,N}`](@ref). It is parameterized by

src/julia-parser.scm

@@ -1330,11 +1330,18 @@
        ((while)  (begin0 (list 'while (parse-cond s) (parse-block s))
                          (expect-end s word)))
        ((for)
-        (let* ((ranges (parse-comma-separated-iters s))
-               (body   (parse-block s)))
-          (expect-end s word)
-          `(for ,(if (length= ranges 1) (car ranges) (cons 'block ranges))
-                ,body)))
+        (let ((r (parse-iteration-spec-or-dotcall s)))
+          (case (car r)
+            ;; for loop
+            ((for)
+             (let* ((ranges (parse-comma-separated-iters-continued s (list (cdr r))))
+                    (body   (parse-block s)))
+               (expect-end s word)
+               `(for ,(if (length= ranges 1) (car ranges) (cons 'block ranges))
+                     ,body)))
+            ;; lazy dotcall
+            (else
+             `(fordot ,(cdr r))))))
 
        ((let)
         (let ((binds (if (memv (peek-token s) '(#\newline #\;))
@@ -1622,6 +1629,12 @@
 
 ;; as above, but allows both "i=r" and "i in r"
 (define (parse-iteration-spec s)
+  (let ((r (parse-iteration-spec-or-dotcall s)))
+    (case (car r)
+      ((for) (cdr r))
+      (else (error "invalid iteration specification")))))
+
+(define (parse-iteration-spec-or-dotcall s)
   (let* ((outer? (if (eq? (peek-token s) 'outer)
                      (begin
                        (take-token s)
@@ -1643,19 +1656,40 @@
                  ;; should be: (error "invalid iteration specification")
                  (parser-depwarn s (string "for " (deparse `(= ,lhs ,rhs)) " " t)
                                  (string "for " (deparse `(= ,lhs ,rhs)) "; " t)))
-             (if outer?
+             (cons
+              'for
+              (if outer?
                  `(= (outer ,lhs) ,rhs)
-                 `(= ,lhs ,rhs))))
+                 `(= ,lhs ,rhs)))))
           ((and (eq? lhs ':) (closing-token? t))
-           ':)
-          (else (error "invalid iteration specification")))))
+           '(for . :))
+
+          (else
+           (if (dotcall? lhs)
+               (cons 'fordot lhs)
+               (error "invalid expression after `for`"))))))
+
+(define (dotcall? ex)
+  (and (pair? ex)
+       (case (car ex)
+         ((call)
+          (equal? (substring (string (cadr ex)) 0 1) "."))
+         ((|.|) (and (pair? (cdr ex))
+                     (pair? (cddr ex))
+                     (pair? (caddr ex))
+                     (eq? (caaddr ex) 'tuple)))
+         (else #f))))
 
 (define (parse-comma-separated-iters s)
-  (let loop ((ranges '()))
-    (let ((r (parse-iteration-spec s)))
-      (case (peek-token s)
-        ((#\,)  (take-token s) (loop (cons r ranges)))
-        (else   (reverse! (cons r ranges)))))))
+  (parse-comma-separated-iters-continued s (list (parse-iteration-spec s))))
+
+(define (parse-comma-separated-iters-continued s ranges)
+  (case (peek-token s)
+    ((#\,)
+     (take-token s)
+     (parse-comma-separated-iters-continued s (cons (parse-iteration-spec s) ranges)))
+    (else
+     (reverse! ranges))))
 
 (define (parse-space-separated-exprs s)
   (with-space-sensitive

src/julia-syntax.scm

@@ -1642,10 +1642,7 @@
         `(block ,@stmts ,nuref))
       expr))
 
-; lazily fuse nested calls to expr == f.(args...) into a single broadcast call,
-; or a broadcast! call if lhs is non-null.
-(define (expand-fuse-broadcast lhs rhs)
-  (define (fuse? e) (and (pair? e) (eq? (car e) 'fuse)))
+(define (expand-lazy-broadcast rhs)
   (define (dot-to-fuse e (top #f)) ; convert e == (. f (tuple args)) to (fuse f args)
     (define (make-fuse f args) ; check for nested (fuse f args) exprs and combine
       (define (split-kwargs args) ; return (cons keyword-args positional-args) extracted from args
@@ -1682,7 +1679,15 @@
                    (list '^ (car x) (expand-forms `(call (call (core apply_type) (top Val) ,(cadr x))))))
                  (make-fuse f x)))
             e)))
-  (let ((e (dot-to-fuse rhs #t)) ; an expression '(fuse func args) if expr is a dot call
+  (dot-to-fuse rhs #t))
+
+(define (fuse? e) (and (pair? e) (eq? (car e) 'fuse)))
+
+; lazily fuse nested calls to expr == f.(args...) into a single broadcast call,
+; or a broadcast! call if lhs is non-null.
+(define (expand-fuse-broadcast lhs rhs)
+  (let ((e ; an expression '(fuse func args) if expr is a dot call
+         (expand-lazy-broadcast rhs))
         (lhs-view (ref-to-view lhs))) ; x[...] expressions on lhs turn in to view(x, ...) to update x in-place
     (if (fuse? e)
         ; expanded to a fuse op call
@@ -1839,6 +1844,13 @@
    (lambda (e)
      (expand-fuse-broadcast (cadr e) (caddr e)))
 
+   'fordot
+   (lambda (e)
+     (let ((x (expand-lazy-broadcast (cadr e))))
+       (if (fuse? x)
+           (expand-forms (cdr x))
+           (error "non-dot call after `for`"))))
+
    '|<:|
    (lambda (e) (expand-forms `(call |<:| ,@(cdr e))))
    '|>:|

test/broadcast.jl

@@ -182,6 +182,14 @@ let x = [1, 3.2, 4.7],
     @test atan.(α, y') == broadcast(atan, α, y')
 end
 
+@testset "for f.(args...) syntax (#19198, #31088)" begin
+    let bc = for (1:3).^2
+        @test bc isa Broadcast.Broadcasted
+        @test sum(bc) == 14
+    end
+    @test sum(for (1:3).^2) == 14
+end
+
 # issue 14725
 let a = Number[2, 2.0, 4//2, 2+0im] / 2
     @test eltype(a) == Number

test/syntax.jl

@@ -1839,3 +1839,66 @@ end
 # issue #31404
 f31404(a, b; kws...) = (a, b, kws.data)
 @test f31404(+, (Type{T} where T,); optimize=false) === (+, (Type,), (optimize=false,))
+
+# lazy dot call
+@test Meta.parse("for f.(x)") == Expr(:fordot, :(f.(x)))
+@test Meta.parse("for f.(g(x))") == Expr(:fordot, :(f.(g(x))))
+@test Meta.parse("for .+ x") == Expr(:fordot, :(.+ x))
+@test Meta.parse("for .+ g(x)") == Expr(:fordot, :(.+ g(x)))
+
+@test Meta.parse("""
+for x in for f.(xs)
+end
+""") == Expr(:for,
+             Expr(:(=), :x, Expr(:fordot, :(f.(xs)))),
+             Expr(:block, LineNumberNode(2, :none)))
+
+@test Meta.parse("""
+for x in for f.(xs), y in for g.(ys)
+end
+""") == Expr(:for,
+             Expr(:block,
+                  Expr(:(=), :x, Expr(:fordot, :(f.(xs)))),
+                  Expr(:(=), :y, Expr(:fordot, :(g.(ys))))),
+             Expr(:block, LineNumberNode(2, :none)))
+
+@testset for valid in [
+        "for f.(x)"
+        "for f.(g(x))"
+        "for .+ x"
+        "for .+ g(x)"
+        "for f.(args[1], args[2])"
+        "for f.(args...)"
+        ]
+    @test Meta.lower(@__MODULE__, Meta.parse(valid)).head == :thunk
+end
+
+@testset for invalid in [
+        "for x"
+        "for f(x)"
+        "for f(g.(x))"
+        "for + x"
+        "for + f.(x))"
+        ]
+    @test_throws ParseError Meta.parse(invalid)
+end
+
+@testset for code in [
+        "for f.(x)"
+        "for f.(g(x))"
+        "for (.+)(x)"
+        "for (.+)(g(x))"
+        ]
+    @test string(Meta.parse(code)) == code
+end
+
+@testset for nondot in [
+        :(f(x))
+        :(f(g.(x)))
+        :(+f.(x))
+        1
+        quote end
+        ]
+    @test Meta.lower(@__MODULE__, Expr(:fordot, nondot)) ==
+        Expr(:error, "non-dot call after `for`")
+end