Initial script string-hash support (#576)

* script: Add string hash value support * script: Fix incorrect identifier token string * script: Add string support to lexer * script: Support parsing strings * script: Output text representation of string if present * script: Add string parse test
BastianBlokland · Sep 9, 2023 · c81dea4 · c81dea4
1 parent 38df4fa
commit c81dea4
Show file tree

Hide file tree

Showing 13 changed files with 237 additions and 67 deletions.
diff --git a/apps/utilities/repl.c b/apps/utilities/repl.c
@@ -34,6 +34,7 @@ static TtyFgColor repl_token_color(const ScriptTokenType tokenType) {
   case ScriptTokenType_Error:
     return TtyFgColor_BrightRed;
   case ScriptTokenType_Number:
+  case ScriptTokenType_String:
     return TtyFgColor_Yellow;
   case ScriptTokenType_Identifier:
     return TtyFgColor_Magenta;

diff --git a/libs/debug/src/brain.c b/libs/debug/src/brain.c
@@ -189,6 +189,11 @@ static bool memory_draw_entity(UiCanvasComp* canvas, ScriptVal* value) {
   return false;
 }
 
+static bool memory_draw_string(UiCanvasComp* canvas, ScriptVal* value) {
+  ui_label(canvas, script_val_str_scratch(*value));
+  return false;
+}
+
 static bool memory_draw_value(UiCanvasComp* canvas, ScriptVal* value) {
   switch (script_type(*value)) {
   case ScriptType_Null:
@@ -202,6 +207,8 @@ static bool memory_draw_value(UiCanvasComp* canvas, ScriptVal* value) {
     return memory_draw_vector3(canvas, value);
   case ScriptType_Entity:
     return memory_draw_entity(canvas, value);
+  case ScriptType_String:
+    return memory_draw_string(canvas, value);
   case ScriptType_Count:
     break;
   }

diff --git a/libs/script/include/script_error.h b/libs/script/include/script_error.h
@@ -5,6 +5,7 @@ typedef enum {
   ScriptError_InvalidChar,
   ScriptError_InvalidUtf8,
   ScriptError_KeyEmpty,
+  ScriptError_UnterminatedString,
   ScriptError_RecursionLimitExceeded,
   ScriptError_MissingPrimaryExpression,
   ScriptError_InvalidPrimaryExpression,

diff --git a/libs/script/include/script_lex.h b/libs/script/include/script_lex.h
@@ -36,6 +36,7 @@ typedef enum {
   ScriptTokenType_Number,       // 42.1337
   ScriptTokenType_Identifier,   // foo
   ScriptTokenType_Key,          // $bar
+  ScriptTokenType_String,       // "Hello World"
   ScriptTokenType_If,           // if
   ScriptTokenType_Else,         // else
   ScriptTokenType_Error,        //
@@ -48,6 +49,7 @@ typedef struct {
     f64         val_number;
     StringHash  val_identifier;
     StringHash  val_key;
+    StringHash  val_string;
     ScriptError val_error;
   };
 } ScriptToken;

diff --git a/libs/script/include/script_val.h b/libs/script/include/script_val.h
@@ -12,6 +12,7 @@ typedef enum {
   ScriptType_Bool,
   ScriptType_Vector3,
   ScriptType_Entity,
+  ScriptType_String,
 
   ScriptType_Count,
 } ScriptType;
@@ -25,6 +26,7 @@ typedef union {
   bool        unsafeBool;
   GeoVector   unsafeVector;
   EcsEntityId unsafeEntity;
+  StringHash  unsafeStringHash;
 } ScriptVal;
 
 ASSERT(sizeof(ScriptVal) == 16, "Expected ScriptVal's size to be 128 bits");
@@ -45,6 +47,7 @@ ScriptVal script_vector3(GeoVector);
 ScriptVal script_vector3_lit(f32 x, f32 y, f32 z);
 ScriptVal script_entity(EcsEntityId);
 ScriptVal script_entity_or_null(EcsEntityId);
+ScriptVal script_string(StringHash);
 ScriptVal script_time(TimeDuration); // Stored as seconds in a number value.
 
 /**
@@ -54,6 +57,7 @@ f64          script_get_number(ScriptVal, f64 fallback);
 bool         script_get_bool(ScriptVal, bool fallback);
 GeoVector    script_get_vector3(ScriptVal, GeoVector fallback);
 EcsEntityId  script_get_entity(ScriptVal, EcsEntityId fallback);
+StringHash   script_get_string(ScriptVal, StringHash fallback);
 TimeDuration script_get_time(ScriptVal, TimeDuration fallback);
 
 /**

diff --git a/libs/script/src/error.c b/libs/script/src/error.c
@@ -6,6 +6,7 @@ static const String g_errorStrs[] = {
     string_static("InvalidChar"),
     string_static("InvalidUtf8"),
     string_static("KeyEmpty"),
+    string_static("UnterminatedString"),
     string_static("RecursionLimitExceeded"),
     string_static("MissingPrimaryExpression"),
     string_static("InvalidPrimaryExpression"),

diff --git a/libs/script/src/lex.c b/libs/script/src/lex.c
@@ -56,13 +56,32 @@ static bool script_is_word_seperator(const u8 c) {
   }
 }
 
+static bool script_is_string_end(const u8 c) {
+  switch (c) {
+  case '\0':
+  case '\n':
+  case '\r':
+  case '"':
+    return true;
+  default:
+    return false;
+  }
+}
+
 static u32 script_scan_word_end(const String str) {
   u32 end = 0;
   for (; end < str.size && !script_is_word_seperator(*string_at(str, end)); ++end)
     ;
   return end;
 }
 
+static u32 script_scan_string_end(const String str) {
+  u32 end = 0;
+  for (; end < str.size && !script_is_string_end(*string_at(str, end)); ++end)
+    ;
+  return end;
+}
+
 static String script_consume_word_or_char(const String str) {
   diag_assert(!string_is_empty(str));
   return string_consume(str, math_max(script_scan_word_end(str), 1));
@@ -99,6 +118,28 @@ static String script_lex_key(String str, StringTable* stringtable, ScriptToken*
   return string_consume(str, end);
 }
 
+static String script_lex_string(String str, StringTable* stringtable, ScriptToken* out) {
+  diag_assert(*string_begin(str) == '"');
+  str = string_consume(str, 1); // Skip the leading '"'.
+
+  const u32 end = script_scan_string_end(str);
+  if (UNLIKELY(end == str.size || *string_at(str, end) != '"')) {
+    *out = script_token_err(ScriptError_UnterminatedString);
+    return str;
+  }
+
+  const String val = string_slice(str, 0, end);
+  if (UNLIKELY(!utf8_validate(val))) {
+    *out = script_token_err(ScriptError_InvalidUtf8);
+    return str;
+  }
+  const StringHash valHash = stringtable ? stringtable_add(stringtable, val) : string_hash(val);
+
+  out->type       = ScriptTokenType_String;
+  out->val_string = valHash;
+  return string_consume(str, end + 1); // + 1 for the closing '"'.
+}
+
 static String script_lex_identifier(String str, ScriptToken* out) {
   const u32 end = script_scan_word_end(str);
   diag_assert(end);
@@ -216,6 +257,8 @@ String script_lex(String str, StringTable* stringtable, ScriptToken* out) {
       return script_lex_number_positive(str, out);
     case '$':
       return script_lex_key(str, stringtable, out);
+    case '"':
+      return script_lex_string(str, stringtable, out);
     case ' ':
     case '\n':
     case '\r':
@@ -245,6 +288,8 @@ bool script_token_equal(const ScriptToken* a, const ScriptToken* b) {
     return a->val_identifier == b->val_identifier;
   case ScriptTokenType_Key:
     return a->val_key == b->val_key;
+  case ScriptTokenType_String:
+    return a->val_string == b->val_string;
   case ScriptTokenType_Error:
     return a->val_error == b->val_error;
   default:
@@ -317,9 +362,11 @@ String script_token_str_scratch(const ScriptToken* token) {
   case ScriptTokenType_Number:
     return fmt_write_scratch("{}", fmt_float(token->val_number));
   case ScriptTokenType_Identifier:
-    return fmt_write_scratch("${}", fmt_int(token->val_identifier, .base = 16));
+    return fmt_write_scratch("{}", fmt_int(token->val_identifier, .base = 16));
   case ScriptTokenType_Key:
     return fmt_write_scratch("${}", fmt_int(token->val_key, .base = 16));
+  case ScriptTokenType_String:
+    return fmt_write_scratch("#{}", fmt_int(token->val_string, .base = 16));
   case ScriptTokenType_If:
     return string_lit("if");
   case ScriptTokenType_Else:

diff --git a/libs/script/src/read.c b/libs/script/src/read.c
@@ -422,6 +422,8 @@ static ScriptReadResult read_expr_primary(ScriptReadContext* ctx) {
    */
   case ScriptTokenType_Number:
     return script_expr(script_add_value(ctx->doc, script_number(token.val_number)));
+  case ScriptTokenType_String:
+    return script_expr(script_add_value(ctx->doc, script_string(token.val_string)));
   /**
    * Memory access.
    */