Initial python branch

build.gradle.kts

@@ -116,6 +116,16 @@ repositories {
             artifact("/[organisation].[module]_[revision].[ext]")
         }
     }
+
+    maven {
+        url = uri("https://oss.sonatype.org/content/repositories/snapshots")
+        mavenContent {
+            snapshotsOnly()
+        }
+        content {
+            includeGroup("io.github.oxisto")
+        }
+    }    
 }
 
 tasks.withType<GenerateModuleMetadata> {
@@ -178,6 +188,8 @@ dependencies {
     // CDT
     api("org.eclipse.cdt", "core", versions["cdt"])
 
+    api("io.github.oxisto", "reticulated-python", "0.2-SNAPSHOT")
+
     // JUnit
     testImplementation("org.junit.jupiter", "junit-jupiter-api", versions["junit5"])
     testImplementation("org.junit.jupiter", "junit-jupiter-params", versions["junit5"])

src/main/java/de/fraunhofer/aisec/cpg/frontends/LanguageFrontendFactory.java

@@ -29,26 +29,29 @@
 import de.fraunhofer.aisec.cpg.TranslationConfiguration;
 import de.fraunhofer.aisec.cpg.frontends.cpp.CXXLanguageFrontend;
 import de.fraunhofer.aisec.cpg.frontends.java.JavaLanguageFrontend;
+import de.fraunhofer.aisec.cpg.frontends.python.PythonLanguageFrontend;
 import java.util.List;
-import org.checkerframework.checker.nullness.qual.Nullable;
 
 public class LanguageFrontendFactory {
 
   private static final List<String> JAVA_EXTENSIONS = List.of(".java");
   private static final List<String> CXX_EXTENSIONS = List.of(".h", ".c", ".cpp", ".cc");
+  private static final List<String> PYTHON_EXTENSIONS = List.of(".py");
 
   // hide ctor
   private LanguageFrontendFactory() {}
 
-  @Nullable
-  public static LanguageFrontend getFrontend(String fileType, TranslationConfiguration config) {
-
+  public static LanguageFrontend getFrontend(String fileType, TranslationConfiguration config)
+      throws TranslationException {
     if (JAVA_EXTENSIONS.contains(fileType)) {
       return new JavaLanguageFrontend(config);
     } else if (CXX_EXTENSIONS.contains(fileType)) {
       return new CXXLanguageFrontend(config);
+    } else if (PYTHON_EXTENSIONS.contains(fileType)) {
+      return new PythonLanguageFrontend(config);
     } else {
-      return null;
+      throw new TranslationException(
+          String.format("No language frontend found for extension %s ", fileType));
     }
   }
 }

src/main/java/de/fraunhofer/aisec/cpg/frontends/python/DefinitionHandler.java

@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2019, Fraunhofer AISEC. All rights reserved.
+ *
+ *  $$$$$$\  $$$$$$$\   $$$$$$\
+ * $$  __$$\ $$  __$$\ $$  __$$\
+ * $$ /  \__|$$ |  $$ |$$ /  \__|
+ * $$ |      $$$$$$$  |$$ |$$$$\
+ * $$ |      $$  ____/ $$ |\_$$ |
+ * $$ |  $$\ $$ |      $$ |  $$ |
+ * \$$$$$   |$$ |      \$$$$$   |
+ *  \______/ \__|       \______/
+ *
+ */
+
+package de.fraunhofer.aisec.cpg.frontends.python;
+
+import de.fraunhofer.aisec.cpg.frontends.Handler;
+import de.fraunhofer.aisec.cpg.graph.CompoundStatement;
+import de.fraunhofer.aisec.cpg.graph.Declaration;
+import de.fraunhofer.aisec.cpg.graph.FunctionDeclaration;
+import de.fraunhofer.aisec.cpg.graph.NodeBuilder;
+import de.fraunhofer.aisec.cpg.graph.ParamVariableDeclaration;
+import de.fraunhofer.aisec.cpg.graph.Type;
+import io.github.oxisto.reticulated.ast.statement.Definition;
+import io.github.oxisto.reticulated.ast.statement.FunctionDefinition;
+import io.github.oxisto.reticulated.ast.statement.Parameter;
+import java.util.ArrayList;
+import java.util.List;
+
+/** Transforms python definitions into declarations. */
+public class DefinitionHandler extends Handler<Declaration, Definition, PythonLanguageFrontend> {
+
+  public DefinitionHandler(PythonLanguageFrontend lang) {
+    super(Declaration::new, lang);
+
+    this.map.put(FunctionDefinition.class, this::handleFunctionDefinition);
+  }
+
+  private Declaration handleFunctionDefinition(Definition def) {
+    FunctionDefinition func = def.asFunctionDefinition();
+
+    FunctionDeclaration declaration =
+        NodeBuilder.newFunctionDeclaration(func.getId().getName(), null);
+
+    this.lang.getScopeManager().enterScope(declaration);
+
+    // build parameters
+    List<ParamVariableDeclaration> parameters = new ArrayList<>();
+    for (Parameter parameter : func.getParameterList()) {
+      String name = parameter.getId().getName();
+      Type type = Type.UNKNOWN;
+
+      // TODO: resolve parameter/type
+
+      ParamVariableDeclaration param = NodeBuilder.newMethodParameterIn(name, type, false, null);
+      parameters.add(param);
+    }
+    declaration.setParameters(parameters);
+
+    // build function body
+    CompoundStatement body = this.lang.getSuiteHandler().handle(func.getSuite());
+
+    declaration.setBody(body);
+
+    this.lang.getScopeManager().leaveScope(declaration);
+
+    return declaration;
+  }
+}

src/main/java/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2019, Fraunhofer AISEC. All rights reserved.
+ *
+ *  $$$$$$\  $$$$$$$\   $$$$$$\
+ * $$  __$$\ $$  __$$\ $$  __$$\
+ * $$ /  \__|$$ |  $$ |$$ /  \__|
+ * $$ |      $$$$$$$  |$$ |$$$$\
+ * $$ |      $$  ____/ $$ |\_$$ |
+ * $$ |  $$\ $$ |      $$ |  $$ |
+ * \$$$$$   |$$ |      \$$$$$   |
+ *  \______/ \__|       \______/
+ *
+ */
+
+package de.fraunhofer.aisec.cpg.frontends.python;
+
+import de.fraunhofer.aisec.cpg.frontends.Handler;
+import de.fraunhofer.aisec.cpg.graph.CallExpression;
+import de.fraunhofer.aisec.cpg.graph.NodeBuilder;
+import io.github.oxisto.reticulated.ast.expression.Call;
+import io.github.oxisto.reticulated.ast.expression.Expression;
+import io.github.oxisto.reticulated.ast.expression.Identifier;
+
+public class ExpressionHandler
+    extends Handler<de.fraunhofer.aisec.cpg.graph.Expression, Expression, PythonLanguageFrontend> {
+
+  public ExpressionHandler(PythonLanguageFrontend lang) {
+    super(de.fraunhofer.aisec.cpg.graph.Expression::new, lang);
+
+    this.map.put(Call.class, this::handleCall);
+  }
+
+  private CallExpression handleCall(Expression expression) {
+    Call call = expression.asCall();
+
+    Identifier id = call.getPrimary().asIdentifier();
+
+    String name = id.getName();
+    String fqn = name;
+
+    CallExpression callExpression = NodeBuilder.newCallExpression(name, fqn, null);
+
+    return callExpression;
+  }
+}

src/main/java/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguageFrontend.java

@@ -0,0 +1,107 @@
+/*
+ * Copyright (c) 2019, Fraunhofer AISEC. All rights reserved.
+ *
+ *  $$$$$$\  $$$$$$$\   $$$$$$\
+ * $$  __$$\ $$  __$$\ $$  __$$\
+ * $$ /  \__|$$ |  $$ |$$ /  \__|
+ * $$ |      $$$$$$$  |$$ |$$$$\
+ * $$ |      $$  ____/ $$ |\_$$ |
+ * $$ |  $$\ $$ |      $$ |  $$ |
+ * \$$$$$   |$$ |      \$$$$$   |
+ *  \______/ \__|       \______/
+ *
+ */
+
+package de.fraunhofer.aisec.cpg.frontends.python;
+
+import de.fraunhofer.aisec.cpg.TranslationConfiguration;
+import de.fraunhofer.aisec.cpg.frontends.LanguageFrontend;
+import de.fraunhofer.aisec.cpg.frontends.TranslationException;
+import de.fraunhofer.aisec.cpg.graph.Region;
+import de.fraunhofer.aisec.cpg.graph.TranslationUnitDeclaration;
+import io.github.oxisto.reticulated.ParserResult;
+import io.github.oxisto.reticulated.PythonParser;
+import io.github.oxisto.reticulated.ast.FileInput;
+import io.github.oxisto.reticulated.ast.statement.Definition;
+import io.github.oxisto.reticulated.ast.statement.Statement;
+import java.io.File;
+import org.checkerframework.checker.nullness.qual.NonNull;
+
+/**
+ * The language frontend for translating python language into the graph. It uses antlr to parse the
+ * actual source code into an AST.
+ */
+public class PythonLanguageFrontend extends LanguageFrontend {
+
+  private StatementHandler statementHandler = new StatementHandler(this);
+  private StatementListHandler statementListHandler = new StatementListHandler(this);
+  private DefinitionHandler definitionHandler = new DefinitionHandler(this);
+  private SuiteHandler suiteHandler = new SuiteHandler(this);
+  private ExpressionHandler expressionHandler = new ExpressionHandler(this);
+  private SimpleStatementHandler simpleStatementHandler = new SimpleStatementHandler(this);
+
+  public PythonLanguageFrontend(TranslationConfiguration config) {
+    super(config, "");
+  }
+
+  @Override
+  public TranslationUnitDeclaration parse(File file) throws TranslationException {
+    PythonParser app = new PythonParser();
+    ParserResult result = app.parse(file.getPath());
+    FileInput input = result.getRoot();
+
+    TranslationUnitDeclaration tu = new TranslationUnitDeclaration();
+
+    for (Statement ctx : input.getStatements()) {
+      // now things get a little tricky, since python does not distinguish between statements and
+      // declarations, but the python parser has an utility class called 'Definition' to distinguish
+      // class and function definitions from other statements such as 'if' and 'for'.
+      if (ctx instanceof Definition) {
+        tu.add(definitionHandler.handle((Definition) ctx));
+      }
+
+      // additionally, python allows statement on a global level, something we also do not allow, so
+      // we need to put them into a virtual function
+
+    }
+
+    return tu;
+  }
+
+  @Override
+  public <T> String getCodeFromRawNode(T astNode) {
+    return null;
+  }
+
+  @Override
+  public @NonNull <T> Region getRegionFromRawNode(T astNode) {
+    return Region.UNKNOWN_REGION;
+  }
+
+  @Override
+  public <S, T> void setComment(S s, T ctx) {}
+
+  public StatementHandler getStatementHandler() {
+    return statementHandler;
+  }
+
+  public DefinitionHandler getDefinitionHandler() {
+    return definitionHandler;
+  }
+
+  public SuiteHandler getSuiteHandler() {
+    return suiteHandler;
+  }
+
+  public StatementListHandler getStatementListHandler() {
+    return statementListHandler;
+  }
+
+  public ExpressionHandler getExpressionHandler() {
+    return expressionHandler;
+  }
+
+  public SimpleStatementHandler getSimpleStatementHandler() {
+    return simpleStatementHandler;
+  }
+}

src/main/java/de/fraunhofer/aisec/cpg/frontends/python/SimpleStatementHandler.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2019, Fraunhofer AISEC. All rights reserved.
+ *
+ *  $$$$$$\  $$$$$$$\   $$$$$$\
+ * $$  __$$\ $$  __$$\ $$  __$$\
+ * $$ /  \__|$$ |  $$ |$$ /  \__|
+ * $$ |      $$$$$$$  |$$ |$$$$\
+ * $$ |      $$  ____/ $$ |\_$$ |
+ * $$ |  $$\ $$ |      $$ |  $$ |
+ * \$$$$$   |$$ |      \$$$$$   |
+ *  \______/ \__|       \______/
+ *
+ */
+
+package de.fraunhofer.aisec.cpg.frontends.python;
+
+import de.fraunhofer.aisec.cpg.frontends.Handler;
+import de.fraunhofer.aisec.cpg.graph.Statement;
+import io.github.oxisto.reticulated.ast.simple.ExpressionStatement;
+import io.github.oxisto.reticulated.ast.simple.SimpleStatement;
+
+public class SimpleStatementHandler
+    extends Handler<Statement, SimpleStatement, PythonLanguageFrontend> {
+
+  public SimpleStatementHandler(PythonLanguageFrontend lang) {
+    super(Statement::new, lang);
+
+    this.map.put(ExpressionStatement.class, this::handleExpressionStatement);
+  }
+
+  private Statement handleExpressionStatement(SimpleStatement simpleStatement) {
+    ExpressionStatement expressionStatement = simpleStatement.asExpressionStatement();
+
+    // un-wrap it
+    return this.lang.getExpressionHandler().handle(expressionStatement.getExpression());
+  }
+}

src/main/java/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2019, Fraunhofer AISEC. All rights reserved.
+ *
+ *  $$$$$$\  $$$$$$$\   $$$$$$\
+ * $$  __$$\ $$  __$$\ $$  __$$\
+ * $$ /  \__|$$ |  $$ |$$ /  \__|
+ * $$ |      $$$$$$$  |$$ |$$$$\
+ * $$ |      $$  ____/ $$ |\_$$ |
+ * $$ |  $$\ $$ |      $$ |  $$ |
+ * \$$$$$   |$$ |      \$$$$$   |
+ *  \______/ \__|       \______/
+ *
+ */
+
+package de.fraunhofer.aisec.cpg.frontends.python;
+
+import de.fraunhofer.aisec.cpg.frontends.Handler;
+import io.github.oxisto.reticulated.ast.statement.Statement;
+
+public class StatementHandler
+    extends Handler<de.fraunhofer.aisec.cpg.graph.Statement, Statement, PythonLanguageFrontend> {
+
+  public StatementHandler(PythonLanguageFrontend lang) {
+    super(de.fraunhofer.aisec.cpg.graph.Statement::new, lang);
+  }
+}

src/main/java/de/fraunhofer/aisec/cpg/frontends/python/StatementListHandler.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2019, Fraunhofer AISEC. All rights reserved.
+ *
+ *  $$$$$$\  $$$$$$$\   $$$$$$\
+ * $$  __$$\ $$  __$$\ $$  __$$\
+ * $$ /  \__|$$ |  $$ |$$ /  \__|
+ * $$ |      $$$$$$$  |$$ |$$$$\
+ * $$ |      $$  ____/ $$ |\_$$ |
+ * $$ |  $$\ $$ |      $$ |  $$ |
+ * \$$$$$   |$$ |      \$$$$$   |
+ *  \______/ \__|       \______/
+ *
+ */
+
+package de.fraunhofer.aisec.cpg.frontends.python;
+
+import de.fraunhofer.aisec.cpg.frontends.Handler;
+import de.fraunhofer.aisec.cpg.graph.Statement;
+import io.github.oxisto.reticulated.ast.simple.SimpleStatement;
+import io.github.oxisto.reticulated.ast.statement.StatementList;
+import java.util.ArrayList;
+import java.util.List;
+
+public class StatementListHandler
+    extends Handler<List<Statement>, StatementList, PythonLanguageFrontend> {
+
+  public StatementListHandler(PythonLanguageFrontend lang) {
+    super(ArrayList::new, lang);
+
+    this.map.put(StatementList.class, this::handleStatementList);
+  }
+
+  private List<Statement> handleStatementList(StatementList statementList) {
+    List<Statement> list = new ArrayList<>();
+
+    for (SimpleStatement node : statementList) {
+      list.add(this.lang.getSimpleStatementHandler().handle(node));
+    }
+
+    return list;
+  }
+}