Handle UTF-16 code unit offsets in file changes API

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "rls-vfs"
-version = "0.6.0"
+version = "0.7.0"
 authors = ["Nick Cameron <[email protected]>"]
 description = "Virtual File System for the RLS"
 license = "Apache-2.0/MIT"
@@ -9,3 +9,4 @@ categories = ["development-tools"]
 
 [dependencies]
 rls-span = "0.4"
+log = "0.4.5"

benches/bench.rs

@@ -7,7 +7,7 @@ extern crate rls_vfs;
 extern crate test;
 
 use rls_span::{Column, Position, Row, Span};
-use rls_vfs::Change;
+use rls_vfs::{Change, VfsSpan};
 use std::fs;
 use std::io::prelude::*;
 use std::path::{Path, PathBuf};
@@ -48,8 +48,7 @@ fn make_change_(path: &Path, start_line: usize, interval: usize) -> Change {
     );
     let buf = (0..LEN).map(|_| txt.to_owned() + "\n").collect::<String>();
     Change::ReplaceText {
-        span: Span::from_positions(start, end, path),
-        len: None,
+        span: VfsSpan::from_usv(Span::from_positions(start, end, path), None),
         text: buf,
     }
 }

src/lib.rs

@@ -1,6 +1,8 @@
 #![feature(type_ascription)]
 
 extern crate rls_span as span;
+#[macro_use]
+extern crate log;
 
 use std::collections::HashMap;
 use std::fmt;
@@ -26,22 +28,69 @@ macro_rules! try_opt_loc {
 
 pub struct Vfs<U = ()>(VfsInternal<RealFileLoader, U>);
 
-type Span = span::Span<span::ZeroIndexed>;
+/// Span of the text to be replaced defined in col/row terms.
+#[derive(Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
+pub struct SpanData {
+    /// Span of the text defined in col/row terms.
+    pub span: span::Span<span::ZeroIndexed>,
+    /// Length in chars of the text. If present,
+    /// used to calculate replacement range instead of
+    /// span's row_end/col_end fields. Needed for editors that
+    /// can't properly calculate the latter fields.
+    /// Span's row_start/col_start are still assumed valid.
+    pub len: Option<u64>,
+}
+
+/// Span of text that VFS can operate with.
+#[derive(Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
+pub enum VfsSpan {
+    /// Span with offsets based on unicode scalar values.
+    UnicodeScalarValue(SpanData),
+    /// Span with offsets based on UTF-16 code units.
+    Utf16CodeUnit(SpanData),
+}
+
+impl VfsSpan {
+    pub fn from_usv(span: span::Span<span::ZeroIndexed>, len: Option<u64>) -> VfsSpan {
+        VfsSpan::UnicodeScalarValue(SpanData { span, len })
+    }
+
+    pub fn from_utf16(span: span::Span<span::ZeroIndexed>, len: Option<u64>) -> VfsSpan {
+        VfsSpan::Utf16CodeUnit(SpanData { span, len })
+    }
+
+    /// Return a UTF-8 byte offset in `s` for a given text unit offset.
+    pub fn byte_in_str(&self, s: &str, c: span::Column<span::ZeroIndexed>) -> Result<usize, Error> {
+        match self {
+            VfsSpan::UnicodeScalarValue(..) => byte_in_str(s, c),
+            VfsSpan::Utf16CodeUnit(..) => byte_in_str_utf16(s, c),
+        }
+    }
+
+    fn as_inner(&self) -> &SpanData {
+        match self {
+            VfsSpan::UnicodeScalarValue(span) => span,
+            VfsSpan::Utf16CodeUnit(span) => span,
+        }
+    }
+
+    pub fn span(&self) -> &span::Span<span::ZeroIndexed> {
+        &self.as_inner().span
+    }
+
+    pub fn len(&self) -> Option<u64> {
+        self.as_inner().len
+    }
+}
 
 #[derive(Debug)]
 pub enum Change {
     /// Create an in-memory image of the file.
     AddFile { file: PathBuf, text: String },
     /// Changes in-memory contents of the previously added file.
     ReplaceText {
-        /// Span of the text to be replaced defined in col/row terms.
-        span: Span,
-        /// Length in chars of the text to be replaced. If present,
-        /// used to calculate replacement range instead of
-        /// span's row_end/col_end fields. Needed for editors that
-        /// can't properly calculate the latter fields.
-        /// Span's row_start/col_start are still assumed valid.
-        len: Option<u64>,
+        /// Span of the text to be replaced.
+        span: VfsSpan,
         /// Text to replace specified text range with.
         text: String,
     },
@@ -51,7 +100,7 @@ impl Change {
     fn file(&self) -> &Path {
         match *self {
             Change::AddFile { ref file, .. } => file.as_ref(),
-            Change::ReplaceText { ref span, .. } => span.file.as_ref(),
+            Change::ReplaceText { ref span, .. } => span.span().file.as_ref(),
         }
     }
 }
@@ -292,6 +341,7 @@ impl<T: FileLoader, U> VfsInternal<T, U> {
     }
 
     fn on_changes(&self, changes: &[Change]) -> Result<(), Error> {
+        trace!("on_changes: {:?}", changes);
         for (file_name, changes) in coalesce_changes(changes) {
             let path = Path::new(file_name);
             {
@@ -652,22 +702,25 @@ impl<U> File<U> {
 
 impl TextFile {
     fn make_change(&mut self, changes: &[&Change]) -> Result<(), Error> {
+        trace!("TextFile::make_change");
         for c in changes {
+            trace!("TextFile::make_change: {:?}", c);
             let new_text = match **c {
                 Change::ReplaceText {
-                    ref span,
-                    ref len,
+                    span: ref vfs_span,
                     ref text,
                 } => {
+                    let (span, len) = (vfs_span.span(), vfs_span.len());
+
                     let range = {
                         let first_line = self.load_line(span.range.row_start)?;
                         let byte_start = self.line_indices[span.range.row_start.0 as usize]
-                            + byte_in_str(first_line, span.range.col_start)? as u32;
+                            + vfs_span.byte_in_str(first_line, span.range.col_start)? as u32;
 
-                        let byte_end = if let &Some(len) = len {
+                        let byte_end = if let Some(len) = len {
                             // if `len` exists, the replaced portion of text
                             // is `len` chars starting from row_start/col_start.
-                            byte_start + byte_in_str(
+                            byte_start + vfs_span.byte_in_str(
                                 &self.text[byte_start as usize..],
                                 span::Column::new_zero_indexed(len as u32),
                             )? as u32
@@ -676,7 +729,7 @@ impl TextFile {
                             // for determining the tail end of replaced text.
                             let last_line = self.load_line(span.range.row_end)?;
                             self.line_indices[span.range.row_end.0 as usize]
-                                + byte_in_str(last_line, span.range.col_end)? as u32
+                                + vfs_span.byte_in_str(last_line, span.range.col_end)? as u32
                         };
 
                         (byte_start, byte_end)
@@ -764,7 +817,7 @@ impl TextFile {
     }
 }
 
-// c is a character offset, returns a byte offset
+/// Return a UTF-8 byte offset in `s` for a given UTF-8 unicode scalar value offset.
 fn byte_in_str(s: &str, c: span::Column<span::ZeroIndexed>) -> Result<usize, Error> {
     // We simulate a null-terminated string here because spans are exclusive at
     // the top, and so that index might be outside the length of the string.
@@ -783,6 +836,27 @@ fn byte_in_str(s: &str, c: span::Column<span::ZeroIndexed>) -> Result<usize, Err
     ));
 }
 
+/// Return a UTF-8 byte offset in `s` for a given UTF-16 code unit offset.
+fn byte_in_str_utf16(s: &str, c: span::Column<span::ZeroIndexed>) -> Result<usize, Error> {
+    let (mut utf8_offset, mut utf16_offset) = (0, 0);
+    let target_utf16_offset = c.0 as usize;
+
+    for chr in s.chars().chain(std::iter::once('\0')) {
+        if utf16_offset > target_utf16_offset {
+            break;
+        } else if utf16_offset == target_utf16_offset {
+            return Ok(utf8_offset);
+        }
+
+        utf8_offset += chr.len_utf8();
+        utf16_offset += chr.len_utf16();
+    }
+
+    return Err(Error::InternalError(
+        "UTF-16 code unit offset is not at `str` char boundary",
+    ));
+}
+
 trait FileLoader {
     fn read<U>(file_name: &Path) -> Result<File<U>, Error>;
     fn write(file_name: &Path, file: &FileKind) -> Result<(), Error>;
@@ -844,3 +918,21 @@ impl FileLoader for RealFileLoader {
         Ok(())
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use span::Column;
+
+    #[test]
+    fn byte_in_str_utf16() {
+        use super::byte_in_str_utf16;
+
+        assert_eq!(
+            '😢'.len_utf8(),
+            byte_in_str_utf16("😢a", Column::new_zero_indexed('😢'.len_utf16() as u32)).unwrap()
+        );
+
+        // 😢 is represented by 2 u16s - we can't index in the middle of a character
+        assert!(byte_in_str_utf16("😢", Column::new_zero_indexed(1)).is_err());
+    }
+}

src/test.rs

@@ -1,9 +1,14 @@
-use super::{VfsInternal, Change, FileLoader, File, FileKind, FileContents,
-            Error, TextFile, make_line_indices};
-use Span;
-use span::{Row, Column};
 use std::path::{Path, PathBuf};
 
+use span::{self, Column, Position, Row};
+
+use super::{
+    make_line_indices, Change, Error, File, FileContents, FileKind, FileLoader, TextFile,
+    VfsInternal, VfsSpan
+};
+
+type Span = span::Span<span::ZeroIndexed>;
+
 struct MockFileLoader;
 
 impl FileLoader for MockFileLoader {
@@ -41,14 +46,16 @@ fn make_change(with_len: bool) -> Change {
         (1, 4, None)
     };
     Change::ReplaceText {
-        span: Span::new(
-            Row::new_zero_indexed(1),
-            Row::new_zero_indexed(row_end),
-            Column::new_zero_indexed(1),
-            Column::new_zero_indexed(col_end),
-            "foo",
+        span: VfsSpan::from_usv(
+            Span::new(
+                Row::new_zero_indexed(1),
+                Row::new_zero_indexed(row_end),
+                Column::new_zero_indexed(1),
+                Column::new_zero_indexed(col_end),
+                "foo",
+            ),
+            len,
         ),
-        len: len,
         text: "foo".to_owned(),
     }
 }
@@ -62,14 +69,16 @@ fn make_change_2(with_len: bool) -> Change {
         (3, 2, None)
     };
     Change::ReplaceText {
-        span: Span::new(
-            Row::new_zero_indexed(2),
-            Row::new_zero_indexed(row_end),
-            Column::new_zero_indexed(4),
-            Column::new_zero_indexed(col_end),
-            "foo",
+        span: VfsSpan::from_usv(
+            Span::new(
+                Row::new_zero_indexed(2),
+                Row::new_zero_indexed(row_end),
+                Column::new_zero_indexed(4),
+                Column::new_zero_indexed(col_end),
+                "foo",
+            ),
+            len,
         ),
-        len: len,
         text: "aye carumba".to_owned(),
     }
 }
@@ -304,4 +313,60 @@ fn test_clear() {
     assert!(vfs.get_cached_files().is_empty());
 }
 
-// TODO test with wide chars
+#[test]
+fn test_wide_utf8() {
+    let vfs = VfsInternal::<MockFileLoader, ()>::new();
+    let changes = [
+        Change::AddFile {
+            file: PathBuf::from("foo"),
+            text: String::from("😢"),
+        },
+        Change::ReplaceText {
+            span: VfsSpan::from_usv(
+                Span::from_positions(
+                    Position::new(Row::new_zero_indexed(0), Column::new_zero_indexed(0)),
+                    Position::new(Row::new_zero_indexed(0), Column::new_zero_indexed(1)),
+                    "foo",
+                ),
+                Some(1),
+            ),
+            text: "".into(),
+        },
+    ];
+
+    vfs.on_changes(&changes).unwrap();
+
+    assert_eq!(
+        vfs.load_file(&Path::new("foo")).unwrap(),
+        FileContents::Text("".to_owned()),
+    );
+}
+
+#[test]
+fn test_wide_utf16() {
+    let vfs = VfsInternal::<MockFileLoader, ()>::new();
+    let changes = [
+        Change::AddFile {
+            file: PathBuf::from("foo"),
+            text: String::from("😢"),
+        },
+        Change::ReplaceText {
+            span: VfsSpan::from_utf16(
+                Span::from_positions(
+                    Position::new(Row::new_zero_indexed(0), Column::new_zero_indexed(0)),
+                    Position::new(Row::new_zero_indexed(0), Column::new_zero_indexed(2)),
+                    "foo",
+                ),
+                Some(2),
+            ),
+            text: "".into(),
+        },
+    ];
+
+    vfs.on_changes(&changes).unwrap();
+
+    assert_eq!(
+        vfs.load_file(&Path::new("foo")).unwrap(),
+        FileContents::Text("".to_owned()),
+    );
+}