From 5a2938c2697e423f4444bf362efdd9a71e30a198 Mon Sep 17 00:00:00 2001 From: Alexandru Calotoiu Date: Mon, 3 Jul 2023 23:43:59 +0200 Subject: [PATCH 1/4] basic allocatable functionality --- dace/frontend/fortran/ast_components.py | 34 +++++++ dace/frontend/fortran/ast_internal_classes.py | 20 ++++ dace/frontend/fortran/ast_transforms.py | 11 +-- dace/frontend/fortran/fortran_parser.py | 96 +++++++++++++------ tests/fortran/allocate_test.py | 51 ++++++++++ 5 files changed, 176 insertions(+), 36 deletions(-) create mode 100644 tests/fortran/allocate_test.py diff --git a/dace/frontend/fortran/ast_components.py b/dace/frontend/fortran/ast_components.py index e386bae23b..e917409017 100644 --- a/dace/frontend/fortran/ast_components.py +++ b/dace/frontend/fortran/ast_components.py @@ -248,6 +248,12 @@ def __init__(self, ast: f03.Program, tables: symbol_table.SymbolTables): "Structure_Constructor": self.structure_constructor, "Component_Spec_List": self.component_spec_list, "Write_Stmt": self.write_stmt, + "Assumed_Shape_Spec_List": self.assumed_shape_spec_list, + "Allocate_Stmt": self.allocate_stmt, + "Allocation_List": self.allocation_list, + "Allocation": self.allocation, + "Allocate_Shape_Spec": self.allocate_shape_spec, + "Allocate_Shape_Spec_List": self.allocate_shape_spec_list, } def list_tables(self): @@ -359,6 +365,30 @@ def array_constructor(self, node: FASTNode): value_list = get_child(children, ast_internal_classes.Ac_Value_List_Node) return ast_internal_classes.Array_Constructor_Node(value_list=value_list.value_list) + def allocate_stmt(self, node: FASTNode): + children = self.create_children(node) + return ast_internal_classes.Allocate_Stmt_Node(allocation_list=children[1]) + + def allocation_list(self, node: FASTNode): + children = self.create_children(node) + return children + + def allocation(self, node: FASTNode): + children = self.create_children(node) + name = get_child(children, ast_internal_classes.Name_Node) + shape = get_child(children, ast_internal_classes.Allocate_Shape_Spec_List) + return ast_internal_classes.Allocation_Node(name=name, shape=shape) + + def allocate_shape_spec_list(self, node: FASTNode): + children = self.create_children(node) + return ast_internal_classes.Allocate_Shape_Spec_List(shape_list=children) + + def allocate_shape_spec(self, node: FASTNode): + children = self.create_children(node) + if len(children) != 2: + raise NotImplementedError("Only simple allocate shape specs are supported") + return children[1] + def structure_constructor(self, node: FASTNode): children = self.create_children(node) name = get_child(children, ast_internal_classes.Type_Name_Node) @@ -490,6 +520,10 @@ def declaration_type_spec(self, node: FASTNode): raise NotImplementedError("Declaration type spec is not supported yet") return node + def assumed_shape_spec_list(self, node: FASTNode): + + return node + def type_declaration_stmt(self, node: FASTNode): #decide if its a intrinsic variable type or a derived type diff --git a/dace/frontend/fortran/ast_internal_classes.py b/dace/frontend/fortran/ast_internal_classes.py index f4dba68fb4..6bdfb61faf 100644 --- a/dace/frontend/fortran/ast_internal_classes.py +++ b/dace/frontend/fortran/ast_internal_classes.py @@ -158,6 +158,26 @@ class Type_Decl_Node(Statement_Node): _fields = () +class Allocate_Shape_Spec_Node(FNode): + _attributes = () + _fields = ('sizes', ) + + +class Allocate_Shape_Spec_List(FNode): + _attributes = () + _fields = ('shape_list', ) + + +class Allocation_Node(FNode): + _attributes = ('name', ) + _fields = ('shape', ) + + +class Allocate_Stmt_Node(FNode): + _attributes = () + _fields = ('allocation_list', ) + + class Symbol_Decl_Node(Statement_Node): _attributes = ( 'name', diff --git a/dace/frontend/fortran/ast_transforms.py b/dace/frontend/fortran/ast_transforms.py index 62c5ad0c7e..7e5cd3bf00 100644 --- a/dace/frontend/fortran/ast_transforms.py +++ b/dace/frontend/fortran/ast_transforms.py @@ -357,10 +357,7 @@ def visit_Array_Subscript_Node(self, node: ast_internal_classes.Array_Subscript_ new_indices.append(ast_internal_classes.Name_Node(name="tmp_index_" + str(tmp))) tmp = tmp + 1 self.count = tmp - return ast_internal_classes.Array_Subscript_Node( - name=node.name, - indices=new_indices, - ) + return ast_internal_classes.Array_Subscript_Node(name=node.name, indices=new_indices) def visit_Execution_Part_Node(self, node: ast_internal_classes.Execution_Part_Node): newbody = [] @@ -543,8 +540,8 @@ def localFunctionStatementEliminator(node: ast_internal_classes.FNode): i.lval, ast_internal_classes.Structure_Constructor_Node): function_statement_name = i.lval.name is_actually_function_statement = False - # In Fortran, function statement are defined as scalar values, - # but called as arrays, so by identifiying that it is called as + # In Fortran, function statement are defined as scalar values, + # but called as arrays, so by identifiying that it is called as # a call_expr or structure_constructor, we also need to match # the specification part and see that it is scalar rather than an array. found = False @@ -562,7 +559,7 @@ def localFunctionStatementEliminator(node: ast_internal_classes.FNode): if is_actually_function_statement: to_change.append([i.lval, i.rval]) new_exec.remove(i) - + else: #There are no function statements after the first one that isn't a function statement break diff --git a/dace/frontend/fortran/fortran_parser.py b/dace/frontend/fortran/fortran_parser.py index 3f3df33997..4180b45371 100644 --- a/dace/frontend/fortran/fortran_parser.py +++ b/dace/frontend/fortran/fortran_parser.py @@ -42,6 +42,7 @@ def __init__(self, ast: ast_components.InternalFortranAst, source: str): self.views = 0 self.libstates = [] self.file_name = source + self.unallocated_arrays = [] self.all_array_names = [] self.last_sdfg_states = {} self.last_loop_continues = {} @@ -64,8 +65,8 @@ def __init__(self, ast: ast_components.InternalFortranAst, source: str): ast_internal_classes.Call_Expr_Node: self.call2sdfg, ast_internal_classes.Program_Node: self.ast2sdfg, ast_internal_classes.Write_Stmt_Node: self.write2sdfg, + ast_internal_classes.Allocate_Stmt_Node: self.allocate2sdfg, } - def get_dace_type(self, type): """ @@ -176,6 +177,44 @@ def basicblock2sdfg(self, node: ast_internal_classes.Execution_Part_Node, sdfg: for i in node.execution: self.translate(i, sdfg) + def allocate2sdfg(self, node: ast_internal_classes.Allocate_Stmt_Node, sdfg: SDFG): + """ + This function is responsible for translating Fortran allocate statements into a SDFG. + :param node: The node to be translated + :param sdfg: The SDFG to which the node should be translated + :note: We pair the allocate with a list of unallocated arrays. + """ + for i in node.allocation_list: + for j in self.unallocated_arrays: + if j[0] == i.name.name and sdfg == j[2]: + datatype = j[1] + transient = j[3] + self.unallocated_arrays.remove(j) + offset_value = -1 + sizes = [] + offset = [] + for j in i.shape.shape_list: + tw = ast_utils.TaskletWriter([], [], sdfg, self.name_mapping) + text = tw.write_code(j) + sizes.append(sym.pystr_to_symbolic(text)) + offset.append(offset_value) + strides = [dat._prod(sizes[:i]) for i in range(len(sizes))] + self.name_mapping[sdfg][i.name.name] = sdfg._find_new_name(i.name.name) + + self.all_array_names.append(self.name_mapping[sdfg][i.name.name]) + if self.contexts.get(sdfg.name) is None: + self.contexts[sdfg.name] = ast_utils.Context(name=sdfg.name) + if i.name.name not in self.contexts[sdfg.name].containers: + self.contexts[sdfg.name].containers.append(i.name.name) + sdfg.add_array(self.name_mapping[sdfg][i.name.name], + shape=sizes, + dtype=datatype, + offset=offset, + strides=strides, + transient=transient) + + #raise NotImplementedError("Fortran allocate statements are not implemented yet") + def write2sdfg(self, node: ast_internal_classes.Write_Stmt_Node, sdfg: SDFG): #TODO implement raise NotImplementedError("Fortran write statements are not implemented yet") @@ -321,7 +360,6 @@ def subroutine2sdfg(self, node: ast_internal_classes.Subroutine_Subprogram_Node, write_names = list(dict.fromkeys([i.name for i in output_vars])) read_names = list(dict.fromkeys([i.name for i in input_vars])) - # Collect the parameters and the function signature to comnpare and link parameters = node.args.copy() @@ -493,7 +531,6 @@ def subroutine2sdfg(self, node: ast_internal_classes.Subroutine_Subprogram_Node, ins_in_new_sdfg.append(self.name_mapping[new_sdfg][local_name.name]) if local_name.name in write_names: outs_in_new_sdfg.append(self.name_mapping[new_sdfg][local_name.name]) - indices = 0 if isinstance(variable_in_call, ast_internal_classes.Array_Subscript_Node): @@ -568,7 +605,7 @@ def subroutine2sdfg(self, node: ast_internal_classes.Subroutine_Subprogram_Node, ins_in_new_sdfg.append(self.name_mapping[new_sdfg][i]) if i in write_names: outs_in_new_sdfg.append(self.name_mapping[new_sdfg][i]) - + array_in_global = self.globalsdfg.arrays[self.name_mapping[self.globalsdfg][i]] if isinstance(array_in_global, Scalar): new_sdfg.add_scalar(self.name_mapping[new_sdfg][i], array_in_global.dtype, transient=False) @@ -580,7 +617,7 @@ def subroutine2sdfg(self, node: ast_internal_classes.Subroutine_Subprogram_Node, transient=False, strides=array_in_global.strides, offset=array_in_global.offset) - # This handles the case where the function is called with wrriten but not read variables found in a module + # This handles the case where the function is called with wrriten but not read variables found in a module for i in not_found_write_names: if i in not_found_read_names: continue @@ -593,7 +630,7 @@ def subroutine2sdfg(self, node: ast_internal_classes.Subroutine_Subprogram_Node, ins_in_new_sdfg.append(self.name_mapping[new_sdfg][i]) if i in write_names: outs_in_new_sdfg.append(self.name_mapping[new_sdfg][i]) - + array = sdfg.arrays[self.name_mapping[sdfg][i]] if isinstance(array_in_global, Scalar): new_sdfg.add_scalar(self.name_mapping[new_sdfg][i], array_in_global.dtype, transient=False) @@ -613,7 +650,7 @@ def subroutine2sdfg(self, node: ast_internal_classes.Subroutine_Subprogram_Node, ins_in_new_sdfg.append(self.name_mapping[new_sdfg][i]) if i in write_names: outs_in_new_sdfg.append(self.name_mapping[new_sdfg][i]) - + array = self.globalsdfg.arrays[self.name_mapping[self.globalsdfg][i]] if isinstance(array_in_global, Scalar): new_sdfg.add_scalar(self.name_mapping[new_sdfg][i], array_in_global.dtype, transient=False) @@ -631,16 +668,16 @@ def subroutine2sdfg(self, node: ast_internal_classes.Subroutine_Subprogram_Node, ins_in_new_sdfg, outs_in_new_sdfg, symbol_mapping=sym_dict) - + # Now adding memlets for i in self.libstates: memlet = "0" if i in write_names: ast_utils.add_memlet_write(substate, self.name_mapping[sdfg][i], internal_sdfg, - self.name_mapping[new_sdfg][i], memlet) + self.name_mapping[new_sdfg][i], memlet) if i in read_names: ast_utils.add_memlet_read(substate, self.name_mapping[sdfg][i], internal_sdfg, - self.name_mapping[new_sdfg][i], memlet) + self.name_mapping[new_sdfg][i], memlet) for i in variables_in_call: @@ -658,7 +695,6 @@ def subroutine2sdfg(self, node: ast_internal_classes.Subroutine_Subprogram_Node, else: raise NameError("Variable name not found: " + ast_utils.get_name(i)) - if not hasattr(var, "shape") or len(var.shape) == 0: memlet = "" elif (len(var.shape) == 1 and var.shape[0] == 1): @@ -687,29 +723,29 @@ def subroutine2sdfg(self, node: ast_internal_classes.Subroutine_Subprogram_Node, if not found: if local_name.name in write_names: ast_utils.add_memlet_write(substate, mapped_name, internal_sdfg, - self.name_mapping[new_sdfg][local_name.name], memlet) + self.name_mapping[new_sdfg][local_name.name], memlet) if local_name.name in read_names: ast_utils.add_memlet_read(substate, mapped_name, internal_sdfg, - self.name_mapping[new_sdfg][local_name.name], memlet) + self.name_mapping[new_sdfg][local_name.name], memlet) for i in addedmemlets: memlet = ast_utils.generate_memlet(ast_internal_classes.Name_Node(name=i), sdfg, self) if local_name.name in write_names: ast_utils.add_memlet_write(substate, self.name_mapping[sdfg][i], internal_sdfg, - self.name_mapping[new_sdfg][i], memlet) + self.name_mapping[new_sdfg][i], memlet) if local_name.name in read_names: ast_utils.add_memlet_read(substate, self.name_mapping[sdfg][i], internal_sdfg, - self.name_mapping[new_sdfg][i], memlet) + self.name_mapping[new_sdfg][i], memlet) for i in globalmemlets: memlet = ast_utils.generate_memlet(ast_internal_classes.Name_Node(name=i), sdfg, self) if local_name.name in write_names: ast_utils.add_memlet_write(substate, self.name_mapping[self.globalsdfg][i], internal_sdfg, - self.name_mapping[new_sdfg][i], memlet) + self.name_mapping[new_sdfg][i], memlet) if local_name.name in read_names: ast_utils.add_memlet_read(substate, self.name_mapping[self.globalsdfg][i], internal_sdfg, - self.name_mapping[new_sdfg][i], memlet) + self.name_mapping[new_sdfg][i], memlet) #Finally, now that the nested sdfg is built and the memlets are added, we can parse the internal of the subroutine and add it to the SDFG. @@ -731,7 +767,6 @@ def subroutine2sdfg(self, node: ast_internal_classes.Subroutine_Subprogram_Node, self.translate(i, new_sdfg) self.translate(node.execution_part, new_sdfg) - def binop2sdfg(self, node: ast_internal_classes.BinOp_Node, sdfg: SDFG): """ This parses binary operations to tasklets in a new state or creates @@ -787,8 +822,8 @@ def binop2sdfg(self, node: ast_internal_classes.BinOp_Node, sdfg: SDFG): output_names_changed = [o_t + "_out" for o_t in output_names] tasklet = ast_utils.add_tasklet(substate, "_l" + str(node.line_number[0]) + "_c" + str(node.line_number[1]), - input_names_tasklet, output_names_changed, "text", node.line_number, - self.file_name) + input_names_tasklet, output_names_changed, "text", node.line_number, + self.file_name) for i, j in zip(input_names, input_names_tasklet): memlet_range = self.get_memlet_range(sdfg, input_vars, i, j) @@ -799,12 +834,11 @@ def binop2sdfg(self, node: ast_internal_classes.BinOp_Node, sdfg: SDFG): memlet_range = self.get_memlet_range(sdfg, output_vars, i, j) ast_utils.add_memlet_write(substate, i, tasklet, k, memlet_range) tw = ast_utils.TaskletWriter(output_names, output_names_changed, sdfg, self.name_mapping, input_names, - input_names_tasklet) + input_names_tasklet) text = tw.write_code(node) tasklet.code = CodeBlock(text, lang.Python) - def call2sdfg(self, node: ast_internal_classes.Call_Expr_Node, sdfg: SDFG): """ This parses function calls to a nested SDFG @@ -879,7 +913,7 @@ def call2sdfg(self, node: ast_internal_classes.Call_Expr_Node, sdfg: SDFG): output_names_changed.append(o_t + "_out") tw = ast_utils.TaskletWriter(output_names_tasklet.copy(), output_names_changed.copy(), sdfg, - self.name_mapping) + self.name_mapping) if not isinstance(rettype, ast_internal_classes.Void) and hasret: special_list_in[retval.name] = pointer(self.get_dace_type(rettype)) special_list_out.append(retval.name + "_out") @@ -896,15 +930,15 @@ def call2sdfg(self, node: ast_internal_classes.Call_Expr_Node, sdfg: SDFG): }, output_names_changed + special_list_out, "text", node.line_number, self.file_name) if libstate is not None: ast_utils.add_memlet_read(substate, self.name_mapping[sdfg][libstate], tasklet, - self.name_mapping[sdfg][libstate] + "_task", "0") + self.name_mapping[sdfg][libstate] + "_task", "0") ast_utils.add_memlet_write(substate, self.name_mapping[sdfg][libstate], tasklet, - self.name_mapping[sdfg][libstate] + "_task_out", "0") + self.name_mapping[sdfg][libstate] + "_task_out", "0") if not isinstance(rettype, ast_internal_classes.Void) and hasret: ast_utils.add_memlet_read(substate, self.name_mapping[sdfg][retval.name], tasklet, retval.name, "0") ast_utils.add_memlet_write(substate, self.name_mapping[sdfg][retval.name], tasklet, - retval.name + "_out", "0") + retval.name + "_out", "0") for i, j in zip(input_names, input_names_tasklet): memlet_range = self.get_memlet_range(sdfg, used_vars, i, j) @@ -938,6 +972,10 @@ def vardecl2sdfg(self, node: ast_internal_classes.Var_Decl_Node, sdfg: SDFG): transient = True # find the type datatype = self.get_dace_type(node.type) + if hasattr(node, "alloc"): + if node.alloc: + self.unallocated_arrays.append([node.name, datatype, sdfg, transient]) + return # get the dimensions if node.sizes is not None: sizes = [] @@ -954,10 +992,10 @@ def vardecl2sdfg(self, node: ast_internal_classes.Var_Decl_Node, sdfg: SDFG): # create and check name - if variable is already defined (function argument and defined in declaration part) simply stop if self.name_mapping[sdfg].get(node.name) is not None: return - + if node.name in sdfg.symbols: return - + self.name_mapping[sdfg][node.name] = sdfg._find_new_name(node.name) if sizes is None: @@ -970,7 +1008,7 @@ def vardecl2sdfg(self, node: ast_internal_classes.Var_Decl_Node, sdfg: SDFG): offset=offset, strides=strides, transient=transient) - + self.all_array_names.append(self.name_mapping[sdfg][node.name]) if self.contexts.get(sdfg.name) is None: self.contexts[sdfg.name] = ast_utils.Context(name=sdfg.name) diff --git a/tests/fortran/allocate_test.py b/tests/fortran/allocate_test.py new file mode 100644 index 0000000000..8d7dbbc856 --- /dev/null +++ b/tests/fortran/allocate_test.py @@ -0,0 +1,51 @@ +# Copyright 2023 ETH Zurich and the DaCe authors. All rights reserved. + +from fparser.common.readfortran import FortranStringReader +from fparser.common.readfortran import FortranFileReader +from fparser.two.parser import ParserFactory +import sys, os +import numpy as np +import pytest + +from dace import SDFG, SDFGState, nodes, dtypes, data, subsets, symbolic +from dace.frontend.fortran import fortran_parser +from fparser.two.symbol_table import SymbolTable +from dace.sdfg import utils as sdutil + +import dace.frontend.fortran.ast_components as ast_components +import dace.frontend.fortran.ast_transforms as ast_transforms +import dace.frontend.fortran.ast_utils as ast_utils +import dace.frontend.fortran.ast_internal_classes as ast_internal_classes + + +def test_fortran_frontend_basic_allocate(): + """ + Tests that the Fortran frontend can parse array accesses and that the accessed indices are correct. + """ + test_string = """ + PROGRAM allocate_test + implicit none + double precision, allocatable :: d(:,:) + allocate(d(4,5)) + CALL allocate_test_function(d) + end + + SUBROUTINE allocate_test_function(d) + double precision d(4,5) + + d(2,1)=5.5 + + END SUBROUTINE allocate_test_function + """ + sdfg = fortran_parser.create_sdfg_from_string(test_string, "allocate_test") + sdfg.simplify(verbose=True) + a = np.full([4,5], 42, order="F", dtype=np.float64) + sdfg(d=a) + assert (a[0,0] == 42) + assert (a[1,0] == 5.5) + assert (a[2,0] == 42) + + +if __name__ == "__main__": + + test_fortran_frontend_basic_allocate() From f3495fa81ad42da0e3445f2a649ed3c05c010b00 Mon Sep 17 00:00:00 2001 From: acalotoiu <61420859+acalotoiu@users.noreply.github.com> Date: Tue, 4 Jul 2023 17:38:05 +0200 Subject: [PATCH 2/4] Update dace/frontend/fortran/ast_components.py Co-authored-by: Philipp Schaad --- dace/frontend/fortran/ast_components.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dace/frontend/fortran/ast_components.py b/dace/frontend/fortran/ast_components.py index e917409017..a66ee5c0d6 100644 --- a/dace/frontend/fortran/ast_components.py +++ b/dace/frontend/fortran/ast_components.py @@ -521,7 +521,6 @@ def declaration_type_spec(self, node: FASTNode): return node def assumed_shape_spec_list(self, node: FASTNode): - return node def type_declaration_stmt(self, node: FASTNode): From 0dba0111da93c2f945f391daae20665b81b05696 Mon Sep 17 00:00:00 2001 From: acalotoiu <61420859+acalotoiu@users.noreply.github.com> Date: Tue, 4 Jul 2023 17:38:17 +0200 Subject: [PATCH 3/4] Update dace/frontend/fortran/fortran_parser.py Co-authored-by: Philipp Schaad --- dace/frontend/fortran/fortran_parser.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dace/frontend/fortran/fortran_parser.py b/dace/frontend/fortran/fortran_parser.py index 4180b45371..6d1be7138a 100644 --- a/dace/frontend/fortran/fortran_parser.py +++ b/dace/frontend/fortran/fortran_parser.py @@ -213,7 +213,6 @@ def allocate2sdfg(self, node: ast_internal_classes.Allocate_Stmt_Node, sdfg: SDF strides=strides, transient=transient) - #raise NotImplementedError("Fortran allocate statements are not implemented yet") def write2sdfg(self, node: ast_internal_classes.Write_Stmt_Node, sdfg: SDFG): #TODO implement From d610ce979db68a67c6f3d36f1cd6f644c6278f1a Mon Sep 17 00:00:00 2001 From: Alexandru Calotoiu Date: Tue, 4 Jul 2023 22:59:34 +0200 Subject: [PATCH 4/4] copyright fix --- tests/fortran/allocate_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fortran/allocate_test.py b/tests/fortran/allocate_test.py index 8d7dbbc856..498c97d932 100644 --- a/tests/fortran/allocate_test.py +++ b/tests/fortran/allocate_test.py @@ -1,4 +1,4 @@ -# Copyright 2023 ETH Zurich and the DaCe authors. All rights reserved. +# Copyright 2019-2023 ETH Zurich and the DaCe authors. All rights reserved. from fparser.common.readfortran import FortranStringReader from fparser.common.readfortran import FortranFileReader