Add support for VIRTUAL_FILE

`VIRTUAL_FILE` declares a file in a new virtual file system.
These files can now be referenced by the `SHADER` commands instead of using inline source.

The DXC compiler will first look in the virtual file system for any `#include`s before falling back to the standard file system.

This allows us to write tests that exercise the compiler and debugger handling of multiple files.

Added `relative_includes_hlsl.amber` which checks that DXC correctly includes relative to the current file, not the root file.

Android.mk

@@ -47,6 +47,7 @@ LOCAL_SRC_FILES:= \
     src/type_parser.cc \
     src/value.cc \
     src/verifier.cc \
+    src/virtual_file_store.cc \
     src/vkscript/command_parser.cc \
     src/vkscript/datum_type_parser.cc \
     src/vkscript/parser.cc \

docs/amber_script.md

@@ -68,8 +68,63 @@ set of data types.
 SET ENGINE_DATA {engine data variable} {value}*
 ```
 
+### Virtual File Store
+
+Each amber script contains a virtual file system that can store files of textual
+data. This lets you bundle multiple source files into a single, hermetic amber
+script file.
+
+Virtual files are declared using the `VIRTUAL_FILE` command:
+
+```groovy
+VIRTUAL_FILE {path}
+ {file-content}
+END
+```
+
+Paths must be unique.
+
+Shaders can directly reference these virtual files for their source. \
+HLSL shaders that `#include` other `.hlsl` files will first check the virtual
+file system, before falling back to the standard file system.
+
 ### Shaders
 
+Shader programs are declared using the `SHADER` command. \
+Shaders can be declared as `PASSTHROUGH`, with inlined source or using source
+from a `VIRTUAL_FILE`.
+
+Pass-through shader:
+
+```groovy
+# Creates a passthrough vertex shader. The shader passes the vec4 at input
+# location 0 through to the `gl_Position`.
+SHADER vertex {shader_name} PASSTHROUGH
+```
+
+Shader using inlined source:
+
+```groovy
+# Creates a shader of |shader_type| with the given |shader_name|. The shader
+# will be of |shader_format|. The shader source then follows and is terminated
+# with the |END| tag.
+SHADER {shader_type} {shader_name} {shader_format}
+{shader_source}
+END
+```
+
+Shader using source from `VIRTUAL_FILE`:
+
+```groovy
+# Creates a shader of |shader_type| with the given |shader_name|. The shader
+# will be of |shader_format|. The shader will use the virtual file with |path|.
+SHADER {shader_type} {shader_name} {shader_format} VIRTUAL_FILE {path}
+```
+
+`{shader_name}` is used to identify the shader to attach to `PIPELINE`s,
+
+`{shader_type}` and `{shader_format}` are described below:
+
 #### Shader Type
  * `vertex`
  * `fragment`
@@ -92,24 +147,11 @@ types, but in that case must only provide a single shader type in the module.
 
 #### Shader Format
  * `GLSL`  (with glslang)
- * `HLSL`  (with dxc or glslang if dxc disabled)  -- future
+ * `HLSL`  (with dxc or glslang if dxc disabled)
  * `SPIRV-ASM` (with spirv-as)
  * `SPIRV-HEX` (decoded straight to SPIR-V)
  * `OPENCL-C` (with clspv)
 
-```groovy
-# Creates a passthrough vertex shader. The shader passes the vec4 at input
-# location 0 through to the `gl_Position`.
-SHADER vertex {shader_name} PASSTHROUGH
-
-# Creates a shader of |shader_type| with the given |shader_name|. The shader
-# will be of |shader_format|. The shader should then be inlined before the
-# |END| tag.
-SHADER {shader_type} {shader_name} {shader_format}
-...
-END
-```
-
 ### Buffers
 
 An AmberScript buffer represents a set of contiguous bits. This can be used for
@@ -471,7 +513,7 @@ RUN {pipeline_name} \
 
 ```groovy
 # Run the given |pipeline_name| which must be a `graphics` pipeline. The
-# grid at |x|, |y|, |width|x|height|, |columns|x|rows| will be rendered. 
+# grid at |x|, |y|, |width|x|height|, |columns|x|rows| will be rendered.
 # Ignores VERTEX_DATA and INDEX_DATA on the given pipeline.
 # For columns, rows of (5, 4) a total of 5*4=20 rectangles will be drawn.
 RUN {pipeline_name} \

src/CMakeLists.txt

@@ -39,6 +39,7 @@ set(AMBER_SOURCES
     type_parser.cc
     value.cc
     verifier.cc
+    virtual_file_store.cc
     vkscript/command_parser.cc
     vkscript/datum_type_parser.cc
     vkscript/parser.cc
@@ -163,6 +164,7 @@ if (${AMBER_ENABLE_TESTS})
     type_parser_test.cc
     type_test.cc
     verifier_test.cc
+    virtual_file_store_test.cc
     vkscript/command_parser_test.cc
     vkscript/datum_type_parser_test.cc
     vkscript/parser_test.cc

src/amberscript/parser.cc

@@ -200,6 +200,8 @@ Result Parser::Parse(const std::string& data) {
       r = ParseStruct();
     } else if (tok == "SAMPLER") {
       r = ParseSampler();
+    } else if (tok == "VIRTUAL_FILE") {
+      r = ParseVirtualFile();
     } else {
       r = Result("unknown token: " + tok);
     }
@@ -370,19 +372,42 @@ Result Parser::ParseShaderBlock() {
 
   shader->SetFormat(format);
 
-  r = ValidateEndOfStatement("SHADER command");
-  if (!r.IsSuccess())
-    return r;
+  token = tokenizer_->PeekNextToken();
+  if (token->IsIdentifier() && token->AsString() == "VIRTUAL_FILE") {
+    tokenizer_->NextToken();  // Skip VIRTUAL_FILE
+
+    token = tokenizer_->NextToken();
+    if (!token->IsIdentifier() && !token->IsString())
+      return Result("expected virtual file path after VIRTUAL_FILE");
 
-  std::string data = tokenizer_->ExtractToNext("END");
-  if (data.empty())
-    return Result("SHADER must not be empty");
+    r = ValidateEndOfStatement("SHADER command");
+    if (!r.IsSuccess())
+      return r;
 
-  shader->SetData(data);
+    auto path = token->AsString();
 
-  token = tokenizer_->NextToken();
-  if (!token->IsIdentifier() || token->AsString() != "END")
-    return Result("SHADER missing END command");
+    std::string data;
+    r = script_->GetVirtualFile(path, &data);
+    if (!r.IsSuccess()) {
+      return r;
+    }
+
+    shader->SetData(data);
+  } else {
+    r = ValidateEndOfStatement("SHADER command");
+    if (!r.IsSuccess())
+      return r;
+
+    std::string data = tokenizer_->ExtractToNext("END");
+    if (data.empty())
+      return Result("SHADER must not be empty");
+
+    shader->SetData(data);
+
+    token = tokenizer_->NextToken();
+    if (!token->IsIdentifier() || token->AsString() != "END")
+      return Result("SHADER missing END command");
+  }
 
   r = script_->AddShader(std::move(shader));
   if (!r.IsSuccess())
@@ -2896,5 +2921,25 @@ Result Parser::ParseTolerances(std::vector<Probe::Tolerance>* tolerances) {
   return {};
 }
 
+Result Parser::ParseVirtualFile() {
+  auto token = tokenizer_->NextToken();
+  if (!token->IsIdentifier() && !token->IsString())
+    return Result("invalid virtual file path");
+
+  auto path = token->AsString();
+
+  auto r = ValidateEndOfStatement("VIRTUAL_FILE command");
+  if (!r.IsSuccess())
+    return r;
+
+  auto data = tokenizer_->ExtractToNext("END");
+
+  token = tokenizer_->NextToken();
+  if (!token->IsIdentifier() || token->AsString() != "END")
+    return Result("VIRTUAL_FILE missing END command");
+
+  return script_->AddVirtualFile(path, data);
+}
+
 }  // namespace amberscript
 }  // namespace amber

src/amberscript/parser.h

@@ -97,6 +97,8 @@ class Parser : public amber::Parser {
                      Format* fmt,
                      std::vector<Value>* values);
 
+  Result ParseVirtualFile();
+
   std::unique_ptr<Tokenizer> tokenizer_;
   std::vector<std::unique_ptr<Command>> command_list_;
 };

src/amberscript/parser_shader_test.cc

@@ -235,6 +235,54 @@ END)";
   EXPECT_EQ("2: extra parameters after SHADER command: INVALID", r.Error());
 }
 
+TEST_F(AmberScriptParserTest, ShaderVirtualFile) {
+  std::string in = R"(#!amber
+VIRTUAL_FILE my_shader.hlsl
+My shader source
+END
+
+SHADER vertex my_shader HLSL VIRTUAL_FILE my_shader.hlsl
+)";
+
+  Parser parser;
+  Result r = parser.Parse(in);
+  ASSERT_EQ(r.Error(), "");
+
+  auto script = parser.GetScript();
+  auto shader = script->GetShader("my_shader");
+  ASSERT_TRUE(shader != nullptr);
+  auto source = shader->GetData();
+  ASSERT_EQ("My shader source\n", shader->GetData());
+}
+
+TEST_F(AmberScriptParserTest, VirtualFileDuplicatePath) {
+  std::string in = R"(#!amber
+VIRTUAL_FILE my.file
+Blah
+END
+
+VIRTUAL_FILE my.file
+Blah
+END
+)";
+
+  Parser parser;
+  Result r = parser.Parse(in);
+  ASSERT_EQ(r.Error(), "8: Virtual file 'my.file' already declared");
+}
+
+TEST_F(AmberScriptParserTest, VirtualFileEmptyPath) {
+  std::string in = R"(#!amber
+VIRTUAL_FILE ""
+Blah
+END
+)";
+
+  Parser parser;
+  Result r = parser.Parse(in);
+  ASSERT_EQ(r.Error(), "4: Virtual file path was empty");
+}
+
 struct ShaderTypeData {
   const char* name;
   ShaderType type;
@@ -315,6 +363,7 @@ TEST_P(AmberScriptParserShaderFormatTest, ShaderFormats) {
   EXPECT_EQ(test_data.format, shader->GetFormat());
   EXPECT_EQ(shader_result, shader->GetData());
 }
+
 INSTANTIATE_TEST_SUITE_P(
     AmberScriptParserTestsShaderFormat,
     AmberScriptParserShaderFormatTest,