From aa9aaf320de29e7378644b5370bfe6bd3c92e08a Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Sun, 8 Dec 2013 21:12:43 +0100 Subject: [PATCH 01/16] Removed unused variables --- app/src/processing/app/debug/Compiler.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/app/src/processing/app/debug/Compiler.java b/app/src/processing/app/debug/Compiler.java index 55329cd661d..8a38db7e90a 100644 --- a/app/src/processing/app/debug/Compiler.java +++ b/app/src/processing/app/debug/Compiler.java @@ -48,10 +48,6 @@ import processing.core.PApplet; public class Compiler implements MessageConsumer { - static final String BUGS_URL = - _("http://github.com/arduino/Arduino/issues"); - static final String SUPER_BADNESS = - I18n.format(_("Compiler error, please submit this code to {0}"), BUGS_URL); private Sketch sketch; From 82db86daeba2dc76c9c2933d0faec2e726da748a Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Mon, 9 Dec 2013 00:10:36 +0100 Subject: [PATCH 02/16] FileUtils: added function to recursively find files with given extension --- app/src/processing/app/helpers/FileUtils.java | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/app/src/processing/app/helpers/FileUtils.java b/app/src/processing/app/helpers/FileUtils.java index 592d2dc74e6..e12fd1fbb9c 100644 --- a/app/src/processing/app/helpers/FileUtils.java +++ b/app/src/processing/app/helpers/FileUtils.java @@ -1,6 +1,7 @@ package processing.app.helpers; import java.io.*; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Random; @@ -188,4 +189,71 @@ public static String readFileToString(File file) throws IOException { } } } + + /** + * Returns true if the given file has any of the given extensions. + * @param file + * File whose name to look at + * @param extensions + * Extensions to consider (just the extension, without the + * dot). Should all be lowercase, case insensitive matching + * is used. + */ + public static boolean hasExtension(File file, String... extensions) { + return hasExtension(file, Arrays.asList(extensions)); + } + + public static boolean hasExtension(File file, List extensions) { + String pieces[] = file.getName().split("\\."); + if (pieces.length < 2) + return false; + + String extension = pieces[pieces.length - 1]; + + return extensions.contains(extension.toLowerCase()); + + } + + /** + * Recursively find all files in a folder with the specified + * extension. Excludes hidden files and folders and + * source control folders. + * + * @param folder + * Folder to look into + * @param recursive + * true will recursively find all files in sub-folders + * @param extensions + * A list of file extensions to search (just the extension, + * without the dot). Should all be lowercase, case + * insensitive matching is used. If no extensions are + * passed, all files are returned. + * @return + */ + public static List listFiles(File folder, boolean recursive, + String... extensions) { + return listFiles(folder, recursive, Arrays.asList(extensions)); + } + + public static List listFiles(File folder, boolean recursive, + List extensions) { + List result = new ArrayList(); + + for (File file : folder.listFiles()) { + if (isSCCSOrHiddenFile(file)) + continue; + + if (file.isDirectory()) { + if (recursive) + result.addAll(listFiles(file, true, extensions)); + continue; + } + + if (extensions.isEmpty() || hasExtension(file, extensions)) + result.add(file); + } + return result; + } + + } From 8e1e7457e68d762b0425acf268d26bceff2e9b1d Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Tue, 8 Jul 2014 14:32:29 +0200 Subject: [PATCH 03/16] Let Sketch.getExtensions() return a List This simplifies upcoming changes. --- app/src/processing/app/Sketch.java | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/app/src/processing/app/Sketch.java b/app/src/processing/app/Sketch.java index 8cd95d7c4c3..ef2eefdf1de 100644 --- a/app/src/processing/app/Sketch.java +++ b/app/src/processing/app/Sketch.java @@ -170,7 +170,7 @@ protected void load() throws IOException { code = new SketchCode[list.length]; - String[] extensions = getExtensions(); + List extensions = getExtensions(); for (String filename : list) { // Ignoring the dot prefix files is especially important to avoid files @@ -1850,11 +1850,7 @@ public boolean isDefaultExtension(String what) { * extensions. */ public boolean validExtension(String what) { - String[] ext = getExtensions(); - for (int i = 0; i < ext.length; i++) { - if (ext[i].equals(what)) return true; - } - return false; + return getExtensions().contains(what); } @@ -1874,8 +1870,8 @@ public List getHiddenExtensions() { /** * Returns a String[] array of proper extensions. */ - public String[] getExtensions() { - return new String[] { "ino", "pde", "c", "cpp", "h" }; + public List getExtensions() { + return Arrays.asList("ino", "pde", "c", "cpp", "h"); } From 3a089e5a5d71c096af5afcaf71c48c05d76dc9db Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Tue, 8 Jul 2014 17:27:38 +0200 Subject: [PATCH 04/16] Make SketchCode.isExtension work for multiple extensions --- app/src/processing/app/SketchCode.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/app/src/processing/app/SketchCode.java b/app/src/processing/app/SketchCode.java index b496755ecb0..800d4732367 100644 --- a/app/src/processing/app/SketchCode.java +++ b/app/src/processing/app/SketchCode.java @@ -25,6 +25,8 @@ package processing.app; import java.io.*; +import java.util.List; +import java.util.Arrays; import javax.swing.text.Document; @@ -156,8 +158,12 @@ public String getExtension() { } - public boolean isExtension(String what) { - return extension.equals(what); + public boolean isExtension(String... extensions) { + return isExtension(Arrays.asList(extensions)); + } + + public boolean isExtension(List extensions) { + return extensions.contains(extension); } From 8519b5d6f16cffc8362d3255508a10f8c546d2a5 Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Tue, 8 Jul 2014 15:26:59 +0200 Subject: [PATCH 05/16] Use SketchCode.isExtension in more places --- app/src/processing/app/EditorHeader.java | 2 +- app/src/processing/app/Sketch.java | 16 +++------------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/app/src/processing/app/EditorHeader.java b/app/src/processing/app/EditorHeader.java index eef679f2ac9..f21228984f4 100644 --- a/app/src/processing/app/EditorHeader.java +++ b/app/src/processing/app/EditorHeader.java @@ -176,7 +176,7 @@ public void paintComponent(Graphics screen) { for (int i = 0; i < sketch.getCodeCount(); i++) { SketchCode code = sketch.getCode(i); - String codeName = sketch.hideExtension(code.getExtension()) ? + String codeName = code.isExtension(sketch.getHiddenExtensions()) ? code.getPrettyName() : code.getFileName(); // if modified, add the li'l glyph next to the name diff --git a/app/src/processing/app/Sketch.java b/app/src/processing/app/Sketch.java index ef2eefdf1de..220b0370828 100644 --- a/app/src/processing/app/Sketch.java +++ b/app/src/processing/app/Sketch.java @@ -425,7 +425,7 @@ protected void nameCode(String newName) { if (renamingCode && currentIndex == 0) { for (int i = 1; i < codeCount; i++) { if (sanitaryName.equalsIgnoreCase(code[i].getPrettyName()) && - code[i].getExtension().equalsIgnoreCase("cpp")) { + code[i].isExtension("cpp")) { Base.showMessage(_("Nope"), I18n.format( _("You can't rename the sketch to \"{0}\"\n" + @@ -836,7 +836,7 @@ protected boolean saveAs() throws IOException { // resaved (with the same name) to another location/folder. for (int i = 1; i < codeCount; i++) { if (newName.equalsIgnoreCase(code[i].getPrettyName()) && - code[i].getExtension().equalsIgnoreCase("cpp")) { + code[i].isExtension("cpp")) { Base.showMessage(_("Nope"), I18n.format( _("You can't save the sketch as \"{0}\"\n" + @@ -1819,21 +1819,11 @@ public boolean isReadOnly() { // Breaking out extension types in order to clean up the code, and make it // easier for other environments (like Arduino) to incorporate changes. - - /** - * True if the specified extension should be hidden when shown on a tab. - * For Processing, this is true for .pde files. (Broken out for subclasses.) - */ - public boolean hideExtension(String what) { - return getHiddenExtensions().contains(what); - } - - /** * True if the specified code has the default file extension. */ public boolean hasDefaultExtension(SketchCode code) { - return code.getExtension().equals(getDefaultExtension()); + return code.isExtension(getDefaultExtension()); } From 83d0a5e2dc116b7e9e19fa527e8661b48cb22296 Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Tue, 8 Jul 2014 15:35:38 +0200 Subject: [PATCH 06/16] Don't store the extension in SketchCode Nobody was using it anymore, except for checking against specific extensions, which is easily done against the filename itself. This prepares for some simplification of Sketch.load next. --- app/src/processing/app/Sketch.java | 12 ++++++------ app/src/processing/app/SketchCode.java | 17 ++++------------- 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/app/src/processing/app/Sketch.java b/app/src/processing/app/Sketch.java index 220b0370828..21e85c0500f 100644 --- a/app/src/processing/app/Sketch.java +++ b/app/src/processing/app/Sketch.java @@ -192,7 +192,7 @@ protected void load() throws IOException { // it would be otherwise possible to sneak in nasty filenames. [0116] if (Sketch.isSanitaryName(base)) { code[codeCount++] = - new SketchCode(new File(folder, filename), extension); + new SketchCode(new File(folder, filename)); } else { editor.console.message(I18n.format("File name {0} is invalid: ignored", filename), true, false); } @@ -487,7 +487,7 @@ protected void nameCode(String newName) { } } - if (!current.renameTo(newFile, newExtension)) { + if (!current.renameTo(newFile)) { Base.showWarning(_("Error"), I18n.format( _("Could not rename \"{0}\" to \"{1}\""), @@ -532,7 +532,7 @@ protected void nameCode(String newName) { editor.base.rebuildSketchbookMenus(); } else { // else if something besides code[0] - if (!current.renameTo(newFile, newExtension)) { + if (!current.renameTo(newFile)) { Base.showWarning(_("Error"), I18n.format( _("Could not rename \"{0}\" to \"{1}\""), @@ -558,7 +558,7 @@ protected void nameCode(String newName) { ), e); return; } - SketchCode newCode = new SketchCode(newFile, newExtension); + SketchCode newCode = new SketchCode(newFile); //System.out.println("new code is named " + newCode.getPrettyName() + " " + newCode.getFile()); insertCode(newCode); } @@ -789,7 +789,7 @@ protected boolean renameCodeToInoExtension(File pdeFile) { String pdeName = pdeFile.getPath(); pdeName = pdeName.substring(0, pdeName.length() - 4) + ".ino"; - return c.renameTo(new File(pdeName), "ino"); + return c.renameTo(new File(pdeName)); } return false; } @@ -1076,7 +1076,7 @@ public boolean addFile(File sourceFile) { } if (codeExtension != null) { - SketchCode newCode = new SketchCode(destFile, codeExtension); + SketchCode newCode = new SketchCode(destFile); if (replacement) { replaceCode(newCode); diff --git a/app/src/processing/app/SketchCode.java b/app/src/processing/app/SketchCode.java index 800d4732367..795a106d03e 100644 --- a/app/src/processing/app/SketchCode.java +++ b/app/src/processing/app/SketchCode.java @@ -31,6 +31,7 @@ import javax.swing.text.Document; import static processing.app.I18n._; +import processing.app.helpers.FileUtils; /** @@ -43,9 +44,6 @@ public class SketchCode { /** File object for where this code is located */ private File file; - /** Extension for this file (no dots, and in lowercase). */ - private String extension; - /** Text of the program text for this tab */ private String program; @@ -72,9 +70,8 @@ public class SketchCode { private int preprocOffset; - public SketchCode(File file, String extension) { + public SketchCode(File file) { this.file = file; - this.extension = extension; makePrettyName(); @@ -127,11 +124,10 @@ public boolean accept(File pathname) { } - protected boolean renameTo(File what, String ext) { + protected boolean renameTo(File what) { boolean success = file.renameTo(what); if (success) { file = what; - extension = ext; makePrettyName(); } return success; @@ -153,17 +149,12 @@ public String getPrettyName() { } - public String getExtension() { - return extension; - } - - public boolean isExtension(String... extensions) { return isExtension(Arrays.asList(extensions)); } public boolean isExtension(List extensions) { - return extensions.contains(extension); + return FileUtils.hasExtension(file, extensions); } From 8bece7c7f4f866fbd7fc8874d7ad36f68e5acf7b Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Tue, 8 Jul 2014 15:36:43 +0200 Subject: [PATCH 07/16] Simplify Sketch.load By using FileUtils.listFiles, this function can be greatly simplified, without changing functionality in any way. --- app/src/processing/app/Sketch.java | 48 ++++++------------------------ 1 file changed, 9 insertions(+), 39 deletions(-) diff --git a/app/src/processing/app/Sketch.java b/app/src/processing/app/Sketch.java index 21e85c0500f..9a49caf0af9 100644 --- a/app/src/processing/app/Sketch.java +++ b/app/src/processing/app/Sketch.java @@ -161,50 +161,20 @@ protected void load() throws IOException { codeFolder = new File(folder, "code"); dataFolder = new File(folder, "data"); - // get list of files in the sketch folder - String list[] = folder.list(); - - // reset these because load() may be called after an - // external editor event. (fix for 0099) - codeCount = 0; - - code = new SketchCode[list.length]; - - List extensions = getExtensions(); - - for (String filename : list) { - // Ignoring the dot prefix files is especially important to avoid files - // with the ._ prefix on Mac OS X. (You'll see this with Mac files on - // non-HFS drives, i.e. a thumb drive formatted FAT32.) - if (filename.startsWith(".")) continue; - - // Don't let some wacko name a directory blah.pde or bling.java. - if (new File(folder, filename).isDirectory()) continue; - - // figure out the name without any extension - String base = filename; - // now strip off the .pde and .java extensions - for (String extension : extensions) { - if (base.toLowerCase().endsWith("." + extension)) { - base = base.substring(0, base.length() - (extension.length() + 1)); - - // Don't allow people to use files with invalid names, since on load, - // it would be otherwise possible to sneak in nasty filenames. [0116] - if (Sketch.isSanitaryName(base)) { - code[codeCount++] = - new SketchCode(new File(folder, filename)); - } else { - editor.console.message(I18n.format("File name {0} is invalid: ignored", filename), true, false); - } - } + List tmpcode = new ArrayList(); + for (File file : FileUtils.listFiles(folder, false, getExtensions())) { + if (Sketch.isSanitaryName(file.getName())) { + tmpcode.add(new SketchCode(file)); + } else { + editor.console.message(I18n.format("File name {0} is invalid: ignored", file.getName()), true, false); } } - if (codeCount == 0) + if (tmpcode.isEmpty()) throw new IOException(_("No valid code files found")); - // Remove any code that wasn't proper - code = (SketchCode[]) PApplet.subset(code, 0, codeCount); + codeCount = tmpcode.size(); + code = tmpcode.toArray(new SketchCode[codeCount]); // move the main class to the first tab // start at 1, if it's at zero, don't bother From beb39814199b230cc781d037797999237c4efa93 Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Mon, 7 Jul 2014 15:22:49 +0200 Subject: [PATCH 08/16] Explicitely store a layout type for a library Previously, the useRecursion and srcFolders were filled on library creation, based on the existence of the src folder. Now, a layout variable is set, and the useRecursion() and getSrcFolder() methods change their return value based on the layout in use. --- app/src/processing/app/packages/Library.java | 29 ++++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/app/src/processing/app/packages/Library.java b/app/src/processing/app/packages/Library.java index 9c505fe4e41..bf69c4edd72 100644 --- a/app/src/processing/app/packages/Library.java +++ b/app/src/processing/app/packages/Library.java @@ -23,10 +23,11 @@ public class Library { private String license; private List architectures; private File folder; - private File srcFolder; - private boolean useRecursion; private boolean isLegacy; + private enum LibraryLayout { FLAT, RECURSIVE }; + private LibraryLayout layout; + private static final List MANDATORY_PROPERTIES = Arrays .asList(new String[] { "name", "version", "author", "maintainer", "sentence", "paragraph", "url" }); @@ -82,12 +83,12 @@ private static Library createLibrary(File libFolder) throws IOException { throw new IOException("Missing '" + p + "' from library"); // Check layout - boolean useRecursion; + LibraryLayout layout; File srcFolder = new File(libFolder, "src"); if (srcFolder.exists() && srcFolder.isDirectory()) { // Layout with a single "src" folder and recursive compilation - useRecursion = true; + layout = LibraryLayout.RECURSIVE; File utilFolder = new File(libFolder, "utility"); if (utilFolder.exists() && utilFolder.isDirectory()) { @@ -96,8 +97,7 @@ private static Library createLibrary(File libFolder) throws IOException { } } else { // Layout with source code on library's root and "utility" folders - srcFolder = libFolder; - useRecursion = false; + layout = LibraryLayout.FLAT; } // Warn if root folder contains development leftovers @@ -134,7 +134,6 @@ private static Library createLibrary(File libFolder) throws IOException { Library res = new Library(); res.folder = libFolder; - res.srcFolder = srcFolder; res.name = properties.get("name").trim(); res.version = properties.get("version").trim(); res.author = properties.get("author").trim(); @@ -145,8 +144,8 @@ private static Library createLibrary(File libFolder) throws IOException { res.category = category.trim(); res.license = license.trim(); res.architectures = archs; - res.useRecursion = useRecursion; res.isLegacy = false; + res.layout = layout; return res; } @@ -154,8 +153,7 @@ private static Library createLegacyLibrary(File libFolder) { // construct an old style library Library res = new Library(); res.folder = libFolder; - res.srcFolder = libFolder; - res.useRecursion = false; + res.layout = LibraryLayout.FLAT; res.name = libFolder.getName(); res.architectures = Arrays.asList("*"); res.isLegacy = true; @@ -246,11 +244,18 @@ public String getMaintainer() { } public boolean useRecursion() { - return useRecursion; + return (layout == LibraryLayout.RECURSIVE); } public File getSrcFolder() { - return srcFolder; + switch (layout) { + case FLAT: + return folder; + case RECURSIVE: + return new File(folder, "src"); + default: + return null; // Keep compiler happy :-( + } } public boolean isLegacy() { From 411cc3bcfe34f2196d0dec854409d40abfc733fb Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Mon, 7 Jul 2014 20:01:32 +0200 Subject: [PATCH 09/16] Simplify Compiler.compileFiles Instead of doing three passes over the filesystem to collect source files, it now does a single pass (using the newly introduced FileUtils.listFiles) and later splits between .S, .c and .cpp files. This allows sharing more code between the three file types and allows removing Compiler.findFilesInFolder. Additionally, this splits compileFiles into a version that compiles all files in a folder and one that only compiles selected files. This prepares for later refactoring of the Library class. This has the side-effect of calling isAlreadyCompiled for .S files as well (which didn't happen previously). However, since the assembler command does not produce any .d files, .S files are still always recompiled, just like before. Finally, this has the side effect of handling file extensions in a case insensitive manner during compilation (instead of just during load), which means that e.g. .CPP files are not just loaded, but also compiled. --- app/src/processing/app/debug/Compiler.java | 76 +++++++--------------- 1 file changed, 24 insertions(+), 52 deletions(-) diff --git a/app/src/processing/app/debug/Compiler.java b/app/src/processing/app/debug/Compiler.java index 8a38db7e90a..c119953f60e 100644 --- a/app/src/processing/app/debug/Compiler.java +++ b/app/src/processing/app/debug/Compiler.java @@ -240,35 +240,35 @@ private PreferencesMap createBuildPreferences(String _buildPath, private List compileFiles(File outputPath, File sourcePath, boolean recurse, List includeFolders) throws RunnerException { - List sSources = findFilesInFolder(sourcePath, "S", recurse); - List cSources = findFilesInFolder(sourcePath, "c", recurse); - List cppSources = findFilesInFolder(sourcePath, "cpp", recurse); - List objectPaths = new ArrayList(); + List sourceFiles = FileUtils.listFiles(sourcePath, recurse, "s", "c", "cpp"); + return compileFiles(outputPath, sourcePath, sourceFiles, includeFolders); - for (File file : sSources) { - File objectFile = new File(outputPath, file.getName() + ".o"); - objectPaths.add(objectFile); - String[] cmd = getCommandCompilerS(includeFolders, file, objectFile); - execAsynchronously(cmd); - } - - for (File file : cSources) { - File objectFile = new File(outputPath, file.getName() + ".o"); - File dependFile = new File(outputPath, file.getName() + ".d"); - objectPaths.add(objectFile); - if (isAlreadyCompiled(file, objectFile, dependFile, prefs)) - continue; - String[] cmd = getCommandCompilerC(includeFolders, file, objectFile); - execAsynchronously(cmd); - } + } - for (File file : cppSources) { + private List compileFiles(File outputPath, File sourcePath, + List sourceFiles, List includeFolders) + throws RunnerException { + List objectPaths = new ArrayList(); + for (File file : sourceFiles) { File objectFile = new File(outputPath, file.getName() + ".o"); File dependFile = new File(outputPath, file.getName() + ".d"); objectPaths.add(objectFile); + if (isAlreadyCompiled(file, objectFile, dependFile, prefs)) continue; - String[] cmd = getCommandCompilerCPP(includeFolders, file, objectFile); + + String[] cmd; + if (FileUtils.hasExtension(file, "s")) { + cmd = getCommandCompilerS(includeFolders, file, objectFile); + } else if (FileUtils.hasExtension(file, "c")) { + cmd = getCommandCompilerC(includeFolders, file, objectFile); + } else if (FileUtils.hasExtension(file, "cpp")) { + cmd = getCommandCompilerCPP(includeFolders, file, objectFile); + } else { + // This should not be possible... + throw new RunnerException("Unknown source file extension: " + file.getName()); + } + execAsynchronously(cmd); } @@ -603,35 +603,6 @@ private void createFolder(File folder) throws RunnerException { throw new RunnerException("Couldn't create: " + folder); } - static public List findFilesInFolder(File folder, String extension, - boolean recurse) { - List files = new ArrayList(); - - if (FileUtils.isSCCSOrHiddenFile(folder)) { - return files; - } - - File[] listFiles = folder.listFiles(); - if (listFiles == null) { - return files; - } - - for (File file : listFiles) { - if (FileUtils.isSCCSOrHiddenFile(file)) { - continue; // skip hidden files - } - - if (file.getName().endsWith("." + extension)) - files.add(file); - - if (recurse && file.isDirectory()) { - files.addAll(findFilesInFolder(file, extension, true)); - } - } - - return files; - } - // 1. compile the sketch (already in the buildPath) void compileSketch(List includeFolders) throws RunnerException { File buildPath = prefs.getFile("build.path"); @@ -666,7 +637,8 @@ private void compileLibrary(Library lib, List includeFolders) includeFolders.add(utilityFolder); compileFilesInFolder(libBuildFolder, libFolder, includeFolders); - compileFilesInFolder(utilityBuildFolder, utilityFolder, includeFolders); + if (utilityFolder.isDirectory()) + compileFilesInFolder(utilityBuildFolder, utilityFolder, includeFolders); // other libraries should not see this library's utility/ folder includeFolders.remove(utilityFolder); From 18096056ad2a55b95b18a325d3b397720aff7d1b Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Mon, 7 Jul 2014 20:04:11 +0200 Subject: [PATCH 10/16] Add FileUtils.relativeSubPath This function allows making a path relative. However, unlike the existing FileUtils.relativePath, this function only works to make a path relative to one of its parent paths (e.g. strip a prefix), which allows it to be a lot simpler. --- app/src/processing/app/helpers/FileUtils.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/app/src/processing/app/helpers/FileUtils.java b/app/src/processing/app/helpers/FileUtils.java index e12fd1fbb9c..ed71b6b137c 100644 --- a/app/src/processing/app/helpers/FileUtils.java +++ b/app/src/processing/app/helpers/FileUtils.java @@ -161,6 +161,21 @@ public static String relativePath(String origin, String target) { return relative + target.substring(origin.length() + 1); } + // Return target, relative to target. Differences with relativePath + // above: + // - Paths are not canonicalized and must be absolute + // - Target must be below origin. + public static String relativeSubPath(String origin, String target) { + // Sanity check + if (!target.startsWith(origin + File.separator)) + return null; + return target.substring(origin.length() + 1); + } + + public static String relativeSubPath(File origin, File target) { + return relativeSubPath(origin.getAbsolutePath(), target.getAbsolutePath()); + } + public static String getLinuxPathFrom(File file) { return BACKSLASH.matcher(file.getAbsolutePath()).replaceAll("/"); } From 806f0fe62a8d2d354cc57d808b45bd3e4e2fe041 Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Mon, 7 Jul 2014 20:38:56 +0200 Subject: [PATCH 11/16] Remove Compiler.[recursive]CompileFilesInFolder Since Compiler.compileFiles already knows how to recursively compile files, it seems pointless to keep these around. Because compileFiles uses a one-off recursion to list files to compile at the start, it has to be slightly smarter about creating directories recursively when needed and needs to fiddle with the filenames a bit (so that the directory structure within a new-style library is maintained in the build directory), but those are only minor changes to compileFiles. --- app/src/processing/app/debug/Compiler.java | 28 +++++++--------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/app/src/processing/app/debug/Compiler.java b/app/src/processing/app/debug/Compiler.java index c119953f60e..3246d7dc144 100644 --- a/app/src/processing/app/debug/Compiler.java +++ b/app/src/processing/app/debug/Compiler.java @@ -250,13 +250,15 @@ private List compileFiles(File outputPath, File sourcePath, throws RunnerException { List objectPaths = new ArrayList(); for (File file : sourceFiles) { - File objectFile = new File(outputPath, file.getName() + ".o"); - File dependFile = new File(outputPath, file.getName() + ".d"); + String relative = FileUtils.relativeSubPath(sourcePath, file); + File objectFile = new File(outputPath, relative + ".o"); + File dependFile = new File(outputPath, relative + ".d"); objectPaths.add(objectFile); if (isAlreadyCompiled(file, objectFile, dependFile, prefs)) continue; + createFolder(objectFile.getParentFile()); String[] cmd; if (FileUtils.hasExtension(file, "s")) { cmd = getCommandCompilerS(includeFolders, file, objectFile); @@ -599,7 +601,7 @@ private String[] getCommandCompilerCPP(List includeFolders, private void createFolder(File folder) throws RunnerException { if (folder.isDirectory()) return; - if (!folder.mkdir()) + if (!folder.mkdirs()) throw new RunnerException("Couldn't create: " + folder); } @@ -625,7 +627,7 @@ private void compileLibrary(Library lib, List includeFolders) if (lib.useRecursion()) { // libBuildFolder == {build.path}/LibName // libFolder == {lib.path}/src - recursiveCompileFilesInFolder(libBuildFolder, libFolder, includeFolders); + objectFiles.addAll(compileFiles(libBuildFolder, libFolder, true, includeFolders)); } else { // libFolder == {lib.path}/ @@ -636,29 +638,15 @@ private void compileLibrary(Library lib, List includeFolders) File utilityBuildFolder = new File(libBuildFolder, "utility"); includeFolders.add(utilityFolder); - compileFilesInFolder(libBuildFolder, libFolder, includeFolders); + objectFiles.addAll(compileFiles(libBuildFolder, libFolder, false, includeFolders)); if (utilityFolder.isDirectory()) - compileFilesInFolder(utilityBuildFolder, utilityFolder, includeFolders); + objectFiles.addAll(compileFiles(utilityBuildFolder, utilityFolder, false, includeFolders)); // other libraries should not see this library's utility/ folder includeFolders.remove(utilityFolder); } } - private void recursiveCompileFilesInFolder(File srcBuildFolder, File srcFolder, List includeFolders) throws RunnerException { - compileFilesInFolder(srcBuildFolder, srcFolder, includeFolders); - for (File subFolder : srcFolder.listFiles(new OnlyDirs())) { - File subBuildFolder = new File(srcBuildFolder, subFolder.getName()); - recursiveCompileFilesInFolder(subBuildFolder, subFolder, includeFolders); - } - } - - private void compileFilesInFolder(File buildFolder, File srcFolder, List includeFolders) throws RunnerException { - createFolder(buildFolder); - List objects = compileFiles(buildFolder, srcFolder, false, includeFolders); - objectFiles.addAll(objects); - } - // 3. compile the core, outputting .o files to and then // collecting them into the core.a library file. // Also compiles the variant (if it supplies actual source files), From 9074a4b59b3a392302385601ae3e5de6b99f4e7d Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Mon, 7 Jul 2014 21:18:15 +0200 Subject: [PATCH 12/16] Introduce constants for file extensions This prevents duplicating these lists in multiple places. As a side-effect, .S files are now handled in sketches as well (instead of just in libraries), fixing #1616. --- app/src/processing/app/Base.java | 5 +++++ app/src/processing/app/Sketch.java | 18 ++++++++++-------- app/src/processing/app/debug/Compiler.java | 2 +- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/app/src/processing/app/Base.java b/app/src/processing/app/Base.java index 4b7207780ff..302c4f840ad 100644 --- a/app/src/processing/app/Base.java +++ b/app/src/processing/app/Base.java @@ -67,6 +67,11 @@ public class Base { /** Set true if this a proper release rather than a numbered revision. */ static public boolean RELEASE = false; + // These should remain lowercase, they are matched against lowercased strings + public static final String[] SOURCE_EXTENSIONS = {"s", "c", "cpp"}; + public static final String[] HEADER_EXTENSIONS = {"h"}; + public static final String[] SKETCH_EXTENSIONS = {"ino", "pde"}; + static Map platformNames = new HashMap(); static { platformNames.put(PConstants.WINDOWS, "windows"); diff --git a/app/src/processing/app/Sketch.java b/app/src/processing/app/Sketch.java index 9a49caf0af9..b92878ff63a 100644 --- a/app/src/processing/app/Sketch.java +++ b/app/src/processing/app/Sketch.java @@ -1308,7 +1308,7 @@ public void preprocess(String buildPath, PdePreprocessor preprocessor) throws Ru StringBuffer bigCode = new StringBuffer(); int bigCount = 0; for (SketchCode sc : code) { - if (sc.isExtension("ino") || sc.isExtension("pde")) { + if (sc.isExtension(Base.SKETCH_EXTENSIONS)) { sc.setPreprocOffset(bigCount); // These #line directives help the compiler report errors with // correct the filename and line number (issue 281 & 907) @@ -1368,7 +1368,7 @@ public void preprocess(String buildPath, PdePreprocessor preprocessor) throws Ru // 3. then loop over the code[] and save each .java file for (SketchCode sc : code) { - if (sc.isExtension("c") || sc.isExtension("cpp") || sc.isExtension("h")) { + if (sc.isExtension(Base.SOURCE_EXTENSIONS) || sc.isExtension(Base.HEADER_EXTENSIONS)) { // no pre-processing services necessary for java files // just write the the contents of 'program' to a .java file // into the build directory. uses byte stream and reader/writer @@ -1382,7 +1382,7 @@ public void preprocess(String buildPath, PdePreprocessor preprocessor) throws Ru } // sc.setPreprocName(filename); - } else if (sc.isExtension("ino") || sc.isExtension("pde")) { + } else if (sc.isExtension(Base.SKETCH_EXTENSIONS)) { // The compiler and runner will need this to have a proper offset sc.addPreprocOffset(headerOffset); } @@ -1818,20 +1818,22 @@ public boolean validExtension(String what) { * Returns the default extension for this editor setup. */ public String getDefaultExtension() { - return "ino"; + return Base.SKETCH_EXTENSIONS[0]; } - static private List hiddenExtensions = Arrays.asList("ino", "pde"); - public List getHiddenExtensions() { - return hiddenExtensions; + return Arrays.asList(Base.SKETCH_EXTENSIONS); } /** * Returns a String[] array of proper extensions. */ public List getExtensions() { - return Arrays.asList("ino", "pde", "c", "cpp", "h"); + List res = new ArrayList(); + res.addAll(Arrays.asList(Base.SKETCH_EXTENSIONS)); + res.addAll(Arrays.asList(Base.SOURCE_EXTENSIONS)); + res.addAll(Arrays.asList(Base.HEADER_EXTENSIONS)); + return res; } diff --git a/app/src/processing/app/debug/Compiler.java b/app/src/processing/app/debug/Compiler.java index 3246d7dc144..41dfebdc33d 100644 --- a/app/src/processing/app/debug/Compiler.java +++ b/app/src/processing/app/debug/Compiler.java @@ -240,7 +240,7 @@ private PreferencesMap createBuildPreferences(String _buildPath, private List compileFiles(File outputPath, File sourcePath, boolean recurse, List includeFolders) throws RunnerException { - List sourceFiles = FileUtils.listFiles(sourcePath, recurse, "s", "c", "cpp"); + List sourceFiles = FileUtils.listFiles(sourcePath, recurse, Base.SOURCE_EXTENSIONS); return compileFiles(outputPath, sourcePath, sourceFiles, includeFolders); } From 37c96110c1067f20ec20c077f08964aec71005cd Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Mon, 7 Jul 2014 20:47:27 +0200 Subject: [PATCH 13/16] Move definition of include and source folders into Library class Previously, the Compiler class had some special casing for new-style and old-style libraries and hardcoded source and include paths for those. Now, this info is moved into the Library class, simplifying the Compiler class and keeping the new vs old-style library stuff in one place. --- app/src/processing/app/Base.java | 38 +++-------- app/src/processing/app/Sketch.java | 14 ++-- app/src/processing/app/debug/Compiler.java | 27 ++------ app/src/processing/app/packages/Library.java | 69 ++++++++++++++++++++ 4 files changed, 87 insertions(+), 61 deletions(-) diff --git a/app/src/processing/app/Base.java b/app/src/processing/app/Base.java index 302c4f840ad..bf918de9569 100644 --- a/app/src/processing/app/Base.java +++ b/app/src/processing/app/Base.java @@ -1349,22 +1349,17 @@ public void onBoardOrPortChange() { // Populate importToLibraryTable importToLibraryTable = new HashMap(); for (Library lib : libraries) { - try { - String headers[] = headerListFromIncludePath(lib.getSrcFolder()); - for (String header : headers) { - Library old = importToLibraryTable.get(header); - if (old != null) { - // If a library was already found with this header, keep - // it if the library's name matches the header name. - String name = header.substring(0, header.length() - 2); - if (old.getFolder().getPath().endsWith(name)) - continue; - } - importToLibraryTable.put(header, lib); + List headers = lib.getPublicHeaders(); + for (String header : headers) { + Library old = importToLibraryTable.get(header); + if (old != null) { + // If a library was already found with this header, keep + // it if the library's name matches the header name. + String name = header.substring(0, header.length() - 2); + if (old.getFolder().getPath().endsWith(name)) + continue; } - } catch (IOException e) { - showWarning(_("Error"), I18n - .format("Unable to list header files in {0}", lib.getSrcFolder()), e); + importToLibraryTable.put(header, lib); } } @@ -1768,19 +1763,6 @@ public void actionPerformed(ActionEvent event) { } } - /** - * Given a folder, return a list of the header files in that folder (but not - * the header files in its sub-folders, as those should be included from - * within the header files at the top-level). - */ - static public String[] headerListFromIncludePath(File path) throws IOException { - String[] list = path.list(new OnlyFilesWithExtension(".h")); - if (list == null) { - throw new IOException(); - } - return list; - } - protected void loadHardware(File folder) { if (!folder.isDirectory()) return; diff --git a/app/src/processing/app/Sketch.java b/app/src/processing/app/Sketch.java index b92878ff63a..b126f3a1607 100644 --- a/app/src/processing/app/Sketch.java +++ b/app/src/processing/app/Sketch.java @@ -1073,20 +1073,14 @@ public boolean addFile(File sourceFile) { } - public void importLibrary(Library lib) throws IOException { - importLibrary(lib.getSrcFolder()); - } - /** * Add import statements to the current tab for all of packages inside - * the specified jar file. + * the specified library. */ - public void importLibrary(File jarPath) throws IOException { + public void importLibrary(Library lib) throws IOException { // make sure the user didn't hide the sketch folder ensureExistence(); - String list[] = Base.headerListFromIncludePath(jarPath); - // import statements into the main sketch file (code[0]) // if the current code is a .java file, insert into current //if (current.flavor == PDE) { @@ -1097,9 +1091,9 @@ public void importLibrary(File jarPath) throws IOException { // statement is already in there, but if the user has the import // commented out, then this will be a problem. StringBuffer buffer = new StringBuffer(); - for (int i = 0; i < list.length; i++) { + for (String include : lib.getPublicHeaders()) { buffer.append("#include <"); - buffer.append(list[i]); + buffer.append(include); buffer.append(">\n"); } buffer.append('\n'); diff --git a/app/src/processing/app/debug/Compiler.java b/app/src/processing/app/debug/Compiler.java index 41dfebdc33d..a0d17d8c4be 100644 --- a/app/src/processing/app/debug/Compiler.java +++ b/app/src/processing/app/debug/Compiler.java @@ -94,7 +94,7 @@ public boolean compile(boolean _verbose) throws RunnerException { System.out.println(I18n .format(_("Using library {0} in folder: {1} {2}"), lib.getName(), lib.getFolder(), lib.isLegacy() ? "(legacy)" : "")); - includeFolders.add(lib.getSrcFolder()); + includeFolders.addAll(lib.getPublicIncludeFolders()); } if (verbose) System.out.println(); @@ -623,28 +623,9 @@ private void compileLibrary(Library lib, List includeFolders) throws RunnerException { File libFolder = lib.getSrcFolder(); File libBuildFolder = prefs.getFile(("build.path"), lib.getName()); - - if (lib.useRecursion()) { - // libBuildFolder == {build.path}/LibName - // libFolder == {lib.path}/src - objectFiles.addAll(compileFiles(libBuildFolder, libFolder, true, includeFolders)); - - } else { - // libFolder == {lib.path}/ - // utilityFolder == {lib.path}/utility - // libBuildFolder == {build.path}/LibName - // utilityBuildFolder == {build.path}/LibName/utility - File utilityFolder = new File(libFolder, "utility"); - File utilityBuildFolder = new File(libBuildFolder, "utility"); - - includeFolders.add(utilityFolder); - objectFiles.addAll(compileFiles(libBuildFolder, libFolder, false, includeFolders)); - if (utilityFolder.isDirectory()) - objectFiles.addAll(compileFiles(utilityBuildFolder, utilityFolder, false, includeFolders)); - - // other libraries should not see this library's utility/ folder - includeFolders.remove(utilityFolder); - } + List libIncludeFolders = new ArrayList(includeFolders); + libIncludeFolders.addAll(lib.getPrivateIncludeFolders()); + objectFiles.addAll(compileFiles(libBuildFolder, libFolder, lib.getSourceFiles(), libIncludeFolders)); } // 3. compile the core, outputting .o files to and then diff --git a/app/src/processing/app/packages/Library.java b/app/src/processing/app/packages/Library.java index bf69c4edd72..0c45a7962ff 100644 --- a/app/src/processing/app/packages/Library.java +++ b/app/src/processing/app/packages/Library.java @@ -9,6 +9,7 @@ import processing.app.helpers.FileUtils; import processing.app.helpers.PreferencesMap; +import processing.app.Base; public class Library { @@ -247,6 +248,10 @@ public boolean useRecursion() { return (layout == LibraryLayout.RECURSIVE); } + /** + * Returns the folder that contains the source files (and possibly + * other files as well). + */ public File getSrcFolder() { switch (layout) { case FLAT: @@ -258,6 +263,70 @@ public File getSrcFolder() { } } + /** + * Include paths within this library to use when compiling libraries + * and sketches depending on this library. + */ + public List getPublicIncludeFolders() { + List res = new ArrayList(); + res.add(getSrcFolder()); + return res; + } + + /** + * Include paths within this library to use when compiling this + * library itself (in addition to the public include folders).. + */ + public List getPrivateIncludeFolders() { + List res = new ArrayList(); + switch (layout) { + case FLAT: + res.add(new File(folder, "utility")); + break; + case RECURSIVE: + break; + } + return res; + } + + /** + * Returns the header files that can be included by other libraries + * and the sketch. The paths returned are relative to one of the + * folders returned by getPublicIncludeFolders - e.g. they are what + * would be inside an #include directory in the code. + */ + public List getPublicHeaders() { + List res = new ArrayList(); + for (File folder : getPublicIncludeFolders()) { + List headers = FileUtils.listFiles(folder, false, Base.HEADER_EXTENSIONS); + for (File header : headers) { + res.add(FileUtils.relativeSubPath(folder, header)); + } + } + return res; + } + + /** + * Returns the complete list of source files for this library. All + * source files are guaranteed to be inside the directory returned by + * getSrcFolder(). + */ + public List getSourceFiles() { + List res = new ArrayList(); + switch (layout) { + case FLAT: + res.addAll(FileUtils.listFiles(folder, false, Base.SOURCE_EXTENSIONS)); + File utilityFolder = new File(folder, "utility"); + if (utilityFolder.isDirectory()) + res.addAll(FileUtils.listFiles(utilityFolder, false, Base.SOURCE_EXTENSIONS)); + break; + case RECURSIVE: + res.addAll(FileUtils.listFiles(getSrcFolder(), true, Base.SOURCE_EXTENSIONS)); + break; + } + return res; + } + public boolean isLegacy() { return isLegacy; } From 5bbe358e801a042fc022a9ea3a8ffa2e798194b3 Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Tue, 8 Jul 2014 11:37:10 +0200 Subject: [PATCH 14/16] Remove unused code --- app/src/processing/app/preproc/PdePreprocessor.java | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/app/src/processing/app/preproc/PdePreprocessor.java b/app/src/processing/app/preproc/PdePreprocessor.java index 10e4b3dd592..67217edc48a 100644 --- a/app/src/processing/app/preproc/PdePreprocessor.java +++ b/app/src/processing/app/preproc/PdePreprocessor.java @@ -58,10 +58,6 @@ public class PdePreprocessor { // off the . and anything after it to produce a package name consistently. List programImports; - // imports just from the code folder, treated differently - // than the others, since the imports are auto-generated. - List codeFolderImports; - String program; @@ -104,13 +100,6 @@ public int writePrefix(String program) for (int i = 0; i < pieces.length; i++) programImports.add(pieces[i][1]); // the package name - codeFolderImports = new ArrayList(); -// if (codeFolderPackages != null) { -// for (String item : codeFolderPackages) { -// codeFolderImports.add(item + ".*"); -// } -// } - prototypes = prototypes(program); // store # of prototypes so that line number reporting can be adjusted From aa08d1393440885ace32e567c53e0e493f65fe79 Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Tue, 8 Jul 2014 11:57:51 +0200 Subject: [PATCH 15/16] Move include detection from PdeProcessor/Sketch to the new HeuristicResolver class This offers a more consistent interface to the library autodetection that can next be expanded to also detect inter-library dependencies. As a side effect, the resolver now looks through all sketch code, including any .c, .cpp and .h files. Previously, only the .ino and .pde files were inspected during preprocessing. This fixes #636. This commit is based on code by Christian Maglie. --- app/src/processing/app/Base.java | 27 ++-- app/src/processing/app/Sketch.java | 27 +--- app/src/processing/app/debug/Compiler.java | 15 +- .../app/packages/HeuristicResolver.java | 142 ++++++++++++++++++ .../app/packages/LibraryResolver.java | 53 +++++++ .../app/preproc/PdePreprocessor.java | 23 --- 6 files changed, 213 insertions(+), 74 deletions(-) create mode 100644 app/src/processing/app/packages/HeuristicResolver.java create mode 100644 app/src/processing/app/packages/LibraryResolver.java diff --git a/app/src/processing/app/Base.java b/app/src/processing/app/Base.java index bf918de9569..dfb60194fb4 100644 --- a/app/src/processing/app/Base.java +++ b/app/src/processing/app/Base.java @@ -44,10 +44,11 @@ import processing.app.helpers.FileUtils; import processing.app.helpers.PreferencesMap; import processing.app.helpers.filefilters.OnlyDirs; -import processing.app.helpers.filefilters.OnlyFilesWithExtension; import processing.app.javax.swing.filechooser.FileNameExtensionFilter; +import processing.app.packages.HeuristicResolver; import processing.app.packages.Library; import processing.app.packages.LibraryList; +import processing.app.packages.LibraryResolver; import processing.app.tools.MenuScroller; import processing.app.tools.ZipDeflater; import processing.core.*; @@ -110,7 +111,7 @@ public class Base { static private LibraryList libraries; // maps #included files to their library folder - static Map importToLibraryTable; + static private LibraryResolver libraryResolver; // classpath for all known libraries for p5 // (both those in the p5/libs folder and those with lib subfolders @@ -1346,22 +1347,8 @@ public void onBoardOrPortChange() { showWarning(_("Error"), _("Error loading libraries"), e); } - // Populate importToLibraryTable - importToLibraryTable = new HashMap(); - for (Library lib : libraries) { - List headers = lib.getPublicHeaders(); - for (String header : headers) { - Library old = importToLibraryTable.get(header); - if (old != null) { - // If a library was already found with this header, keep - // it if the library's name matches the header name. - String name = header.substring(0, header.length() - 2); - if (old.getFolder().getPath().endsWith(name)) - continue; - } - importToLibraryTable.put(header, lib); - } - } + // Create library resolver + libraryResolver = new HeuristicResolver(libraries); // Update editors status bar for (Editor editor : editors) @@ -2996,4 +2983,8 @@ public void handleAddLibrary() { public static DiscoveryManager getDiscoveryManager() { return discoveryManager; } + + public static LibraryResolver getLibraryResolver() { + return libraryResolver; + } } diff --git a/app/src/processing/app/Sketch.java b/app/src/processing/app/Sketch.java index b126f3a1607..843f85aa7c8 100644 --- a/app/src/processing/app/Sketch.java +++ b/app/src/processing/app/Sketch.java @@ -34,13 +34,13 @@ import processing.app.helpers.FileUtils; import processing.app.packages.Library; import processing.app.packages.LibraryList; +import processing.app.packages.LibraryResolver; import processing.app.preproc.*; import processing.core.*; import static processing.app.I18n._; import java.io.*; import java.util.*; -import java.util.List; import javax.swing.*; @@ -91,11 +91,6 @@ public class Sketch { /** Class path determined during build. */ private String classPath; - /** - * List of library folders. - */ - private LibraryList importedLibraries; - /** * File inside the build directory that contains the build options * used for the last build. @@ -1348,18 +1343,8 @@ public void preprocess(String buildPath, PdePreprocessor preprocessor) throws Ru ex.printStackTrace(); throw new RunnerException(ex.toString()); } - - // grab the imports from the code just preproc'd - - importedLibraries = new LibraryList(); - for (String item : preprocessor.getExtraImports()) { - Library lib = Base.importToLibraryTable.get(item); - if (lib != null && !importedLibraries.contains(lib)) { - importedLibraries.add(lib); - } - } - - // 3. then loop over the code[] and save each .java file + + // 2. then loop over the code[] and save each .java file for (SketchCode sc : code) { if (sc.isExtension(Base.SOURCE_EXTENSIONS) || sc.isExtension(Base.HEADER_EXTENSIONS)) { @@ -1383,12 +1368,6 @@ public void preprocess(String buildPath, PdePreprocessor preprocessor) throws Ru } } - - public LibraryList getImportedLibraries() { - return importedLibraries; - } - - /** * Map an error from a set of processed .java files back to its location * in the actual sketch. diff --git a/app/src/processing/app/debug/Compiler.java b/app/src/processing/app/debug/Compiler.java index a0d17d8c4be..716bd943fed 100644 --- a/app/src/processing/app/debug/Compiler.java +++ b/app/src/processing/app/debug/Compiler.java @@ -45,6 +45,7 @@ import processing.app.helpers.StringReplacer; import processing.app.helpers.filefilters.OnlyDirs; import processing.app.packages.Library; +import processing.app.packages.LibraryList; import processing.core.PApplet; public class Compiler implements MessageConsumer { @@ -89,7 +90,8 @@ public boolean compile(boolean _verbose) throws RunnerException { includeFolders.add(prefs.getFile("build.core.path")); if (prefs.getFile("build.variant.path") != null) includeFolders.add(prefs.getFile("build.variant.path")); - for (Library lib : sketch.getImportedLibraries()) { + LibraryList libs = Base.getLibraryResolver().findDirectDependencies(sketch); + for (Library lib : libs) { if (verbose) System.out.println(I18n .format(_("Using library {0} in folder: {1} {2}"), lib.getName(), @@ -105,7 +107,7 @@ public boolean compile(boolean _verbose) throws RunnerException { String[] overrides = prefs.get("architecture.override_check").split(","); archs.addAll(Arrays.asList(overrides)); } - for (Library lib : sketch.getImportedLibraries()) { + for (Library lib : libs) { if (!lib.supportsArchitecture(archs)) { System.err.println(I18n .format(_("WARNING: library {0} claims to run on {1} " @@ -124,7 +126,8 @@ public boolean compile(boolean _verbose) throws RunnerException { // 2. compile the libraries, outputting .o files to: // // Doesn't really use configPreferences sketch.setCompilingProgress(40); - compileLibraries(includeFolders); + for (Library lib : libs) + compileLibrary(lib, includeFolders); // 3. compile the core, outputting .o files to and then // collecting them into the core.a library file. @@ -613,12 +616,6 @@ void compileSketch(List includeFolders) throws RunnerException { // 2. compile the libraries, outputting .o files to: // // - void compileLibraries(List includeFolders) throws RunnerException { - for (Library lib : sketch.getImportedLibraries()) { - compileLibrary(lib, includeFolders); - } - } - private void compileLibrary(Library lib, List includeFolders) throws RunnerException { File libFolder = lib.getSrcFolder(); diff --git a/app/src/processing/app/packages/HeuristicResolver.java b/app/src/processing/app/packages/HeuristicResolver.java new file mode 100644 index 00000000000..58310bcba1d --- /dev/null +++ b/app/src/processing/app/packages/HeuristicResolver.java @@ -0,0 +1,142 @@ +/* + * This file is part of Arduino. + * + * Arduino is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As a special exception, you may use this file as part of a free software + * library without restriction. Specifically, if other files instantiate + * templates or use macros or inline functions from this file, or you compile + * this file and link it with other files to produce an executable, this + * file does not by itself cause the resulting executable to be covered by + * the GNU General Public License. This exception does not however + * invalidate any other reasons why the executable file might be covered by + * the GNU General Public License. + * + * Copyright 2013 Arduino LLC (http://www.arduino.cc/) + */ + +package processing.app.packages; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.LinkedList; +import java.util.Map; + +import processing.app.Sketch; +import processing.app.SketchCode; +import processing.app.helpers.FileUtils; +import processing.app.helpers.filefilters.OnlyFilesWithExtension; +import processing.app.preproc.PdePreprocessor; +import processing.core.PApplet; + +/** + * This resolver uses an heuristic approach to resolve dependencies + * without looking into metadata. + */ +public class HeuristicResolver implements LibraryResolver { + + private LibraryList libraries; + private Map importToLibrary; + + public HeuristicResolver(LibraryList _libraries) { + libraries = _libraries; + importToLibrary = new HashMap(); + + // Populate importToLibrary table + for (Library library : libraries) { + for (String header : library.getPublicHeaders()) { + Library old = importToLibrary.get(header); + if (old != null) { + // If a library was already found with this header, keep + // it if the library's name matches the header name. + String name = header.substring(0, header.length() - 2); + if (old.getFolder().getPath().endsWith(name)) + continue; + } + importToLibrary.put(header, library); + } + } + } + + @Override + public LibraryList findDirectDependencies(Sketch sketch) { + SketchCode files[] = sketch.getCode(); + LibraryList result = new LibraryList(); + for (SketchCode code : files) + result.addOrReplaceAll(findSourceDependencies(code.getFile(), null, null)); + return result; + } + + /** + * Inspect headerFile and search for dependencies + * + * @param headerFile + * @param excluded + * @param library + */ + private LibraryList findSourceDependencies(File headerFile, + List excluded, + Library library) { + LibraryList res = new LibraryList(); + + + // Extract #includes from header file + List imports; + String contents; + try { + contents = FileUtils.readFileToString(headerFile); + } catch (IOException e) { + e.printStackTrace(); + return res; + } + + String importRegexp = "^\\s*#include\\s*[<\"](\\S+)[\">]"; + String[][] pieces = PApplet.matchAll(contents, importRegexp); + + // For every #include found... + if (pieces != null) { + for (int i = 0; i < pieces.length; i++) { + String libImport = pieces[i][1]; + + // ...check if the include is not in the exclusion list + if (excluded != null && excluded.contains(libImport)) + continue; + + // ...check if there is a matching library + Library depLib = importToLibrary.get(libImport); + if (depLib == null || depLib == library) + continue; + + // ...that we didn't see before + if (res.contains(depLib)) + continue; + + // add the dependency + res.add(depLib); + } + } + + return res; + } + + @Override + public Library importToLibrary(String h) { + return importToLibrary.get(h); + } + +} diff --git a/app/src/processing/app/packages/LibraryResolver.java b/app/src/processing/app/packages/LibraryResolver.java new file mode 100644 index 00000000000..8008071a00f --- /dev/null +++ b/app/src/processing/app/packages/LibraryResolver.java @@ -0,0 +1,53 @@ +/* + * This file is part of Arduino. + * + * Arduino is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As a special exception, you may use this file as part of a free software + * library without restriction. Specifically, if other files instantiate + * templates or use macros or inline functions from this file, or you compile + * this file and link it with other files to produce an executable, this + * file does not by itself cause the resulting executable to be covered by + * the GNU General Public License. This exception does not however + * invalidate any other reasons why the executable file might be covered by + * the GNU General Public License. + * + * Copyright 2013 Arduino LLC (http://www.arduino.cc/) + */ + +package processing.app.packages; + +import processing.app.Sketch; + +public interface LibraryResolver { + + /** + * Resolve direct dependencies for a sketch + * + * @param sketch + * @return A LibraryList containing the dependencies + */ + public abstract LibraryList findDirectDependencies(Sketch sketch); + + /** + * Returns the Library referenced by the include file name + * + * @param header + * The include file name, for example "SPI.h". + * @return The referenced library + */ + public abstract Library importToLibrary(String header); + +} diff --git a/app/src/processing/app/preproc/PdePreprocessor.java b/app/src/processing/app/preproc/PdePreprocessor.java index 67217edc48a..9d9f1fa85ea 100644 --- a/app/src/processing/app/preproc/PdePreprocessor.java +++ b/app/src/processing/app/preproc/PdePreprocessor.java @@ -53,11 +53,6 @@ public class PdePreprocessor { // the prototypes that are generated by the preprocessor List prototypes; - // these ones have the .* at the end, since a class name might be at the end - // instead of .* which would make trouble other classes using this can lop - // off the . and anything after it to produce a package name consistently. - List programImports; - String program; @@ -90,16 +85,6 @@ public int writePrefix(String program) program = substituteUnicode(program); } - //String importRegexp = "(?:^|\\s|;)(import\\s+)(\\S+)(\\s*;)"; - String importRegexp = "^\\s*#include\\s*[<\"](\\S+)[\">]"; - programImports = new ArrayList(); - - String[][] pieces = PApplet.matchAll(program, importRegexp); - - if (pieces != null) - for (int i = 0; i < pieces.length; i++) - programImports.add(pieces[i][1]); // the package name - prototypes = prototypes(program); // store # of prototypes so that line number reporting can be adjusted @@ -188,14 +173,6 @@ protected void writeProgram(PrintStream out, String program, List protot protected void writeFooter(PrintStream out) throws java.lang.Exception {} - public List getExtraImports() { - return programImports; - } - - - - - /** * Returns the index of the first character that's not whitespace, a comment * or a pre-processor directive. From 2aba55d77657f274df0ac9d5994b0d2b0c8ddada Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Tue, 8 Jul 2014 12:19:08 +0200 Subject: [PATCH 16/16] Automatically detect dependencies of libraries as well Previously, any libraries #included by the sketch were added to the include path and link, but any libraries included by other libraries were not. This meant that all libraries used, even indirectly by other libraries, must be explicitely included in the sketch. This commit changes the dependency resolution to become recursive - it also includes the libraries included by other libraries in the build. This commit is based on code by Christian Maglie and fixes #236. --- app/src/processing/app/debug/Compiler.java | 4 +- .../app/packages/HeuristicResolver.java | 40 +++++++++++++++++++ app/src/processing/app/packages/Library.java | 19 +++++---- .../app/packages/LibraryResolver.java | 24 +++++++++++ 4 files changed, 78 insertions(+), 9 deletions(-) diff --git a/app/src/processing/app/debug/Compiler.java b/app/src/processing/app/debug/Compiler.java index 716bd943fed..640cb3d1514 100644 --- a/app/src/processing/app/debug/Compiler.java +++ b/app/src/processing/app/debug/Compiler.java @@ -90,7 +90,7 @@ public boolean compile(boolean _verbose) throws RunnerException { includeFolders.add(prefs.getFile("build.core.path")); if (prefs.getFile("build.variant.path") != null) includeFolders.add(prefs.getFile("build.variant.path")); - LibraryList libs = Base.getLibraryResolver().findDirectDependencies(sketch); + LibraryList libs = Base.getLibraryResolver().findRecursiveDependencies(sketch); for (Library lib : libs) { if (verbose) System.out.println(I18n @@ -622,7 +622,7 @@ private void compileLibrary(Library lib, List includeFolders) File libBuildFolder = prefs.getFile(("build.path"), lib.getName()); List libIncludeFolders = new ArrayList(includeFolders); libIncludeFolders.addAll(lib.getPrivateIncludeFolders()); - objectFiles.addAll(compileFiles(libBuildFolder, libFolder, lib.getSourceFiles(), libIncludeFolders)); + objectFiles.addAll(compileFiles(libBuildFolder, libFolder, lib.getSourceFiles(false), libIncludeFolders)); } // 3. compile the core, outputting .o files to and then diff --git a/app/src/processing/app/packages/HeuristicResolver.java b/app/src/processing/app/packages/HeuristicResolver.java index 58310bcba1d..ff2b193685f 100644 --- a/app/src/processing/app/packages/HeuristicResolver.java +++ b/app/src/processing/app/packages/HeuristicResolver.java @@ -82,6 +82,46 @@ public LibraryList findDirectDependencies(Sketch sketch) { return result; } + @Override + public LibraryList findRecursiveDependencies(Sketch sketch) { + LibraryList result = findDirectDependencies(sketch); + findRecursiveDependencies(result); + return result; + } + + @Override + public LibraryList findDirectDependencies(Library library) { + List files = library.getSourceFiles(true); + List headers = library.getPublicHeaders(); + LibraryList result = new LibraryList(); + for (File file : files) + result.addOrReplaceAll(findSourceDependencies(file, headers, library)); + return result; + } + + @Override + public LibraryList findRecursiveDependencies(Library library) { + LibraryList result = findDirectDependencies(library); + findRecursiveDependencies(result); + return result; + } + + // Helper function - recursively finds the dependencies of the given + // list of libraries and adds them to the given list. + private void findRecursiveDependencies(LibraryList libs) { + LinkedList todo = new LinkedList(libs); + while (!todo.isEmpty()) { + Library next = todo.pop(); + LibraryList indirects = findDirectDependencies(next); + for (Library indirect : indirects) { + if (!libs.contains(indirect)) { + libs.add(indirect); + todo.add(indirect); + } + } + } + } + /** * Inspect headerFile and search for dependencies * diff --git a/app/src/processing/app/packages/Library.java b/app/src/processing/app/packages/Library.java index 0c45a7962ff..80bada1c733 100644 --- a/app/src/processing/app/packages/Library.java +++ b/app/src/processing/app/packages/Library.java @@ -307,21 +307,26 @@ public List getPublicHeaders() { } /** - * Returns the complete list of source files for this library. All - * source files are guaranteed to be inside the directory returned by - * getSrcFolder(). + * Returns the complete list of source (and optionally header) files + * for this library. All source files are guaranteed to be inside the + * directory returned by getSrcFolder(). */ - public List getSourceFiles() { + public List getSourceFiles(boolean includeHeaders) { List res = new ArrayList(); + List exts = new ArrayList(Arrays.asList(Base.SOURCE_EXTENSIONS)); + if (includeHeaders) + exts.addAll(Arrays.asList(Base.HEADER_EXTENSIONS)); + String extsArray[] = exts.toArray(new String[exts.size()]); + switch (layout) { case FLAT: - res.addAll(FileUtils.listFiles(folder, false, Base.SOURCE_EXTENSIONS)); + res.addAll(FileUtils.listFiles(folder, false, extsArray)); File utilityFolder = new File(folder, "utility"); if (utilityFolder.isDirectory()) - res.addAll(FileUtils.listFiles(utilityFolder, false, Base.SOURCE_EXTENSIONS)); + res.addAll(FileUtils.listFiles(utilityFolder, false, extsArray)); break; case RECURSIVE: - res.addAll(FileUtils.listFiles(getSrcFolder(), true, Base.SOURCE_EXTENSIONS)); + res.addAll(FileUtils.listFiles(getSrcFolder(), true, extsArray)); break; } return res; diff --git a/app/src/processing/app/packages/LibraryResolver.java b/app/src/processing/app/packages/LibraryResolver.java index 8008071a00f..07161e12952 100644 --- a/app/src/processing/app/packages/LibraryResolver.java +++ b/app/src/processing/app/packages/LibraryResolver.java @@ -41,6 +41,30 @@ public interface LibraryResolver { */ public abstract LibraryList findDirectDependencies(Sketch sketch); + /** + * Resolve recursive dependencies for a sketch + * + * @param sketch + * @return A LibraryList containing the dependencies + */ + public abstract LibraryList findRecursiveDependencies(Sketch sketch); + + /** + * Resolve direct dependencies for a library + * + * @param library + * @return A LibraryList containing the dependencies + */ + public abstract LibraryList findDirectDependencies(Library sketch); + + /** + * Resolve recursive dependencies for a library + * + * @param library + * @return A LibraryList containing the dependencies + */ + public abstract LibraryList findRecursiveDependencies(Library library); + /** * Returns the Library referenced by the include file name *