Add latin1 transcoding bindings (#16)

* Latin1 codegen

* Generated

* Basic tests

* No utf32 to latin1

* Fix

* simplify

---------

Co-authored-by: Nugine <nugine@foxmail.com>

codegen/src/api.rs

@@ -153,6 +153,9 @@ fn codegen_count() {
 
 fn codegen_transcoding_length() {
     for_each_transcoding_length(|from, to| {
+        if from == "latin1" && to == "utf32" {
+            return;
+        }
         let from_ch = map_rs_char_type(from);
         let from_doc_name = map_doc_name(from);
         let to_doc_name = map_doc_name(to);
@@ -174,9 +177,14 @@ fn codegen_transcoding_length() {
         g!("#[inline]");
         g!("#[must_use]");
         g!("pub unsafe fn {to}_length_from_{from}(src: &[{from_ch}]) -> usize {{");
-        g!("let len = src.len();");
-        g!("let buf = src.as_ptr();");
-        g!("crate::bindings::simdutf_{to}_length_from_{from}(buf, len)");
+        if is_fixed_length_for_latin1(from, to) {
+            g!("let len = src.len();");
+            g!("crate::bindings::simdutf_{to}_length_from_{from}(len)");
+        } else {
+            g!("let len = src.len();");
+            g!("let buf = src.as_ptr();");
+            g!("crate::bindings::simdutf_{to}_length_from_{from}(buf, len)");
+        }
         g!("}}");
         g!();
     })
@@ -213,6 +221,9 @@ fn codegen_transcoding_convert() {
     });
 
     for_each_transcoding_convert(|from, to| {
+        if from == "latin1" || (from == "utf32" && to == "latin1") {
+            return;
+        }
         let from_ch = map_rs_char_type(from);
         let to_ch = map_rs_char_type(to);
         let from_doc_name = map_doc_name(from);
@@ -239,6 +250,9 @@ fn codegen_transcoding_convert() {
     });
 
     for_each_transcoding_convert(|from, to| {
+        if from == "latin1" || (from == "utf32" && to == "latin1") {
+            return;
+        }
         let from_ch = map_rs_char_type(from);
         let to_ch = map_rs_char_type(to);
         let from_doc_name = map_doc_name(from);

codegen/src/bindings.rs

@@ -39,6 +39,16 @@ pub fn codegen_cpp() {
     });
 
     for_each_transcoding_length(|from, to| {
+        if from == "latin1" && to == "utf32" {
+            return;
+        }
+        if is_fixed_length_for_latin1(from, to) {
+            g!("size_t simdutf_{to}_length_from_{from}(size_t len) {{");
+            g!("    return simdutf::{to}_length_from_{from}(len);");
+            g!("}}");
+            g!();
+            return;
+        }
         let from_ch = map_cpp_char_type(from);
         g!("size_t simdutf_{to}_length_from_{from}(const {from_ch}* buf, size_t len) {{");
         g!("    return simdutf::{to}_length_from_{from}(buf, len);");
@@ -56,6 +66,9 @@ pub fn codegen_cpp() {
     });
 
     for_each_transcoding_convert(|from, to| {
+        if from == "latin1" || (from == "utf32" && to == "latin1") {
+            return;
+        }
         let from_ch = map_cpp_char_type(from);
         let to_ch = map_cpp_char_type(to);
         g!("simdutfrs_result_t \
@@ -68,6 +81,9 @@ pub fn codegen_cpp() {
     });
 
     for_each_transcoding_convert(|from, to| {
+        if from == "latin1" || (from == "utf32" && to == "latin1") {
+            return;
+        }
         let from_ch = map_cpp_char_type(from);
         let to_ch = map_cpp_char_type(to);
         g!("size_t simdutf_convert_valid_{from}_to_{to}(const {from_ch}* src, size_t len, {to_ch}* dst) {{");
@@ -125,6 +141,13 @@ pub fn codegen_rust() {
     g!();
 
     for_each_transcoding_length(|from, to| {
+        if from == "latin1" && to == "utf32" {
+            return;
+        }
+        if is_fixed_length_for_latin1(from, to) {
+            g!("pub fn simdutf_{to}_length_from_{from}(len: usize) -> usize;");
+            return;
+        }
         let from_ch = map_rs_char_type(from);
         g!("pub fn simdutf_{to}_length_from_{from}(buf: *const {from_ch}, len: usize) -> usize;");
     });
@@ -139,6 +162,9 @@ pub fn codegen_rust() {
     g!();
 
     for_each_transcoding_convert(|from, to| {
+        if from == "latin1" || (from == "utf32" && to == "latin1") {
+            return;
+        }
         let from_ch = map_rs_char_type(from);
         let to_ch = map_rs_char_type(to);
         g!("pub fn simdutf_convert_{from}_to_{to}_with_errors\
@@ -147,6 +173,9 @@ pub fn codegen_rust() {
     g!();
 
     for_each_transcoding_convert(|from, to| {
+        if from == "latin1" || (from == "utf32" && to == "latin1") {
+            return;
+        }
         let from_ch = map_rs_char_type(from);
         let to_ch = map_rs_char_type(to);
         g!("pub fn simdutf_convert_valid_{from}_to_{to}\

codegen/src/common.rs

@@ -1,4 +1,4 @@
-pub const ENCODINGS: [&str; 6] = ["ascii", "utf8", "utf16", "utf16be", "utf16le", "utf32"];
+pub const ENCODINGS: [&str; 7] = ["ascii", "utf8", "utf16", "utf16be", "utf16le", "utf32", "latin1"];
 
 pub fn map_cpp_char_type(encoding: &str) -> &str {
     match encoding {
@@ -8,6 +8,7 @@ pub fn map_cpp_char_type(encoding: &str) -> &str {
         "utf16be" => "char16_t",
         "utf16le" => "char16_t",
         "utf32" => "char32_t",
+        "latin1" => "char",
         _ => unimplemented!(),
     }
 }
@@ -20,6 +21,7 @@ pub fn map_rs_char_type(encoding: &str) -> &str {
         "utf16be" => "u16",
         "utf16le" => "u16",
         "utf32" => "u32",
+        "latin1" => "u8",
         _ => unimplemented!(),
     }
 }
@@ -32,19 +34,23 @@ pub fn map_doc_name(encoding: &str) -> &str {
         "utf16be" => "UTF-16BE",
         "utf16le" => "UTF-16LE",
         "utf32" => "UTF-32",
+        "latin1" => "Latin1",
         _ => unimplemented!(),
     }
 }
 
 pub fn for_each_validate(mut f: impl FnMut(&str)) {
     for encoding in ENCODINGS {
+        if encoding == "latin1" {
+            continue;
+        }
         f(encoding);
     }
 }
 
 pub fn for_each_count(mut f: impl FnMut(&str)) {
     for encoding in ENCODINGS {
-        if matches!(encoding, "ascii" | "utf32") {
+        if matches!(encoding, "ascii" | "utf32" | "latin1") {
             continue;
         }
         f(encoding);
@@ -57,6 +63,12 @@ pub fn for_each_transcoding_length(mut f: impl FnMut(&str, &str)) {
             if from == "ascii" || to == "ascii" {
                 continue;
             }
+            if from == "latin1" && (to == "utf16le" || to == "utf16be") {
+                continue;
+            }
+            if (from == "utf16le" || from == "utf16be") && to == "latin1" {
+                continue;
+            }
             if from == to {
                 continue;
             }
@@ -87,3 +99,13 @@ pub fn for_each_transcoding_convert(mut f: impl FnMut(&str, &str)) {
         }
     }
 }
+
+pub fn is_fixed_length_for_latin1(from: &str, to: &str) -> bool {
+    if from == "latin1" && to != "utf8" {
+        return true;
+    }
+    if from != "utf8" && to == "latin1" {
+        return true;
+    }
+    false
+}