Add functions to convert between code-point and code-unit positions

LSP `Position`s use UTF-16 code units for offsets within lines; most
other sane tools (like GHC) use Unicode code points. We need to use the
right one in the right place, otherwise we get issues like
haskell/haskell-language-server#2646.

This is pretty unpleasant, since code points are variable-size, so you
can't do the conversion without having the file text itself.

This PR provides a type for positions using code points (for clients to
use to help them be less confused) and functions for using the VFS to
convert between those and LSP positions.

Author: michaelpj

lsp-types/src/Language/LSP/Types/Location.hs

@@ -11,6 +11,8 @@ import           Language.LSP.Types.Utils
 
 -- ---------------------------------------------------------------------
 
+-- | A position in a document. Note that the character offsets in a line
+-- are given in UTF-16 code units, *not* Unicode code points.
 data Position =
   Position
     { -- | Line position in a document (zero-based).

lsp/src/Language/LSP/VFS.hs

@@ -44,6 +44,11 @@ module Language.LSP.VFS
   , persistFileVFS
   , closeVFS
 
+  -- * Positions and transformations
+  , CodePointPosition (..)
+  , codePointPositionToPosition
+  , positionToCodePointPosition
+
   -- * manipulating the file contents
   , rangeLinesFromVfs
   , PosPrefixInfo(..)
@@ -69,6 +74,7 @@ import           Data.Ord
 import qualified Data.HashMap.Strict as HashMap
 import qualified Data.Map.Strict as Map
 import           Data.Maybe
+import qualified Data.Text.Rope as URope
 import           Data.Text.Utf16.Rope ( Rope )
 import qualified Data.Text.Utf16.Rope as Rope
 import           Data.Text.Prettyprint.Doc
@@ -346,6 +352,49 @@ changeChars logger str start finish new = do
 
 -- ---------------------------------------------------------------------
 
+-- | A position, like a 'J.Position', but where the offsets in the line are measured in Unicode code points
+-- instead of UTF-16 code units.
+data CodePointPosition =
+  CodePointPosition
+    { -- | Line position in a document (zero-based).
+      _line      :: J.UInt
+      -- | Character offset on a line in a document in *code points* (zero-based).
+    , _character :: J.UInt
+    } deriving (Show, Read, Eq, Ord)
+
+-- | Given a virtual file, translate a 'CodePointPosition' in that file into a 'J.Position' in that file.
+--
+-- We need the file itself because this requires translating between code points and code units.
+codePointPositionToPosition :: VirtualFile -> CodePointPosition -> J.Position
+codePointPositionToPosition vFile (CodePointPosition cpl cpc) =
+  let utf16Text = _file_text vFile
+      -- Transcode to a code-point based rope
+      utfText = URope.fromText $ Rope.toText utf16Text
+      -- Split at the given position
+      (utfPrefix, _) = URope.splitAtPosition (URope.Position (fromIntegral cpl) (fromIntegral cpc)) utfText
+      -- Transcode the prefix to a code-unit based rope
+      utf16Prefix = Rope.fromText $ URope.toText utfPrefix
+      -- Get the length of the transcoded prefix
+      (Rope.Position cul cuc) = Rope.lengthAsPosition utf16Prefix
+  in J.Position (fromIntegral cul) (fromIntegral cuc)
+
+-- | Given a virtual file, translate a 'J.Position' in that file into a 'CodePointPosition' in that file.
+-- May fail if the requested position lies inside a code point.
+--
+-- We need the file itself because this requires translating between code unit and code points.
+positionToCodePointPosition :: VirtualFile -> J.Position -> Maybe CodePointPosition
+positionToCodePointPosition vFile (J.Position cul cuc) = do
+  let utf16Text = _file_text vFile
+  -- Split at the given location
+  (utf16Prefix, _) <- Rope.splitAtPosition (Rope.Position (fromIntegral cul) (fromIntegral cuc)) utf16Text
+  -- Transcode the preix to a code-point based rope
+  let utfPrefix = URope.fromText $ Rope.toText utf16Prefix
+      -- Get the length of the transcoded prefix
+      (URope.Position cpl cpc) = URope.lengthAsPosition utfPrefix
+  pure $ CodePointPosition (fromIntegral cpl) (fromIntegral cpc)
+
+-- ---------------------------------------------------------------------
+
 -- TODO:AZ:move this to somewhere sane
 -- | Describes the line at the current cursor position
 data PosPrefixInfo = PosPrefixInfo