chore: vendor nucleo-ext + fork, so we can depend on our changes properly (#3284)

We cannot publish to crates.io without specifying a version, and we
cannot do that without properly forking nucleo. We're shipping
atuin-nucleo, but will likely drop this if we can get our changes
upstream. This is highlighted in the README + manifest, and the original
author is still included.

Originally forked here: https://github.com/atuinsh/nucleo-ext

cc @BinaryMuse - this should just be a vendor + restructure, but would
appreciate the sanity check

## Checks
- [ ] I am happy for maintainers to push small adjustments to this PR,
to speed up the review cycle
- [ ] I have checked that there are no existing pull requests for the
same thing

1 files changed, 44 insertions, 0 deletions

diff --git a/crates/atuin-nucleo/matcher/src/utf32_str/tests.rs b/crates/atuin-nucleo/matcher/src/utf32_str/tests.rs
new file mode 100644
index 00000000..a38c8875
--- /dev/null
+++ b/crates/atuin-nucleo/matcher/src/utf32_str/tests.rs
@@ -0,0 +1,44 @@
+use crate::{Utf32Str, Utf32String};
+
+#[test]
+fn test_utf32str_ascii() {
+    /// Helper function for testing
+    fn expect_ascii(src: &str, is_ascii: bool) {
+        let mut buffer = Vec::new();
+        assert!(Utf32Str::new(src, &mut buffer).is_ascii() == is_ascii);
+        assert!(Utf32String::from(src).slice(..).is_ascii() == is_ascii);
+        assert!(Utf32String::from(src.to_owned()).slice(..).is_ascii() == is_ascii);
+    }
+
+    // ascii
+    expect_ascii("", true);
+    expect_ascii("a", true);
+    expect_ascii("a\nb", true);
+    expect_ascii("\n\r", true);
+
+    // not ascii
+    expect_ascii("aü", false);
+    expect_ascii("au\u{0308}", false);
+
+    // windows-style newline
+    expect_ascii("a\r\nb", false);
+    expect_ascii("ü\r\n", false);
+    expect_ascii("\r\n", false);
+}
+
+#[test]
+fn test_grapheme_truncation() {
+    // ascii is preserved
+    let s = Utf32String::from("ab");
+    assert_eq!(s.slice(..).get(0), 'a');
+    assert_eq!(s.slice(..).get(1), 'b');
+
+    // windows-style newline is truncated to '\n'
+    let s = Utf32String::from("\r\n");
+    assert_eq!(s.slice(..).get(0), '\n');
+
+    // normal graphemes are truncated to the first character
+    let s = Utf32String::from("u\u{0308}\r\n");
+    assert_eq!(s.slice(..).get(0), 'u');
+    assert_eq!(s.slice(..).get(1), '\n');
+}