Support utf16 offsets, and upgrade textdocumentsync to 2

TODO.md

@@ -1,18 +1,6 @@
 # Wishlist of features
 ([X] = complete,  [ ] = planned)
 
-                  code unit count (not to be confused with code points)
-code point #    |utf8 |utf16| offset
-000000 - 00007F |  1  |  1  |   0
-000080 - 0007FF |  2  |  1  |  -1
-000800 - 00FFFF |  3  |  1  |  -2
-010000 - 10FFFF |  4  |  2  |  -2
-
-utf8 chart
-000000 - 00007F	0xxxxxxx	
-000080 - 0007FF	110xxxxx	10xxxxxx	
-000800 - 00FFFF	1110xxxx	10xxxxxx	10xxxxxx	
-010000 - 10FFFF	11110xxx	10xxxxxx	10xxxxxx	10xxxxxx
 My current goal is to work on completions a little bit more.
 
 - [ ] Fix crash-files.test2
@@ -86,6 +74,7 @@ Here is my feature wishlist. I don't expect to ever get all of this done, but th
     - [ ] hide or grey out the `self` in an `a:b` multisym call
 - [ ] Go-to-references
     - [x] lexical scope in the same file
+    - [ ] function names work properly and are counted once
     - [ ] fields
     - [ ] go to references of fields when tables are aliased
     - [ ] global search across other files

src/fennel-ls/compiler.fnl

@@ -40,10 +40,18 @@ later by fennel-ls.language to answer requests from the client."
        (tset self key val)
        val))})
 
+(λ line+byte->range [self file line byte]
+  (let [line (- line 1)
+        ;; TODO think about this further when upstream bug #180 is fixed
+        byte (math.max 0 byte)
+        position (utils.pos->position file.text line byte self.position-encoding)]
+    {:start position :end position}))
+
+
 (λ is-values? [?ast]
   (and (list? ?ast) (= (sym :values) (. ?ast 1))))
 
-(λ compile [{:configuration {: macro-path} : root-uri} file]
+(λ compile [{:configuration {: macro-path} : root-uri &as self} file]
   "Compile the file, and record all the useful information from the compiler into the file object"
   ;; The useful information being recorded:
   (let [definitions-by-scope (doto {} (setmetatable has-tables-mt))
@@ -193,7 +201,7 @@ later by fennel-ls.language to answer requests from the client."
         ;; This cannot be done through the :fn feature of the compiler plugin system
         ;; because it needs to be called *before* the body of the function is processed.
         ;; TODO check if hashfn needs to be here
-        (where (or [(= -fn-)] [(= -lambda-)] [(= -λ-)] false))  ;; TODO, this false pattern should not ever match, and should be removed once I update fennel
+        (where (or [(= -fn-)] [(= -lambda-)] [(= -λ-)]))
         (define-function ast scope)
         (where [(= -require-) _modname])
         (tset require-calls ast true)
@@ -210,8 +218,8 @@ later by fennel-ls.language to answer requests from the client."
           (= 1 (msg:find "expected at least one pattern/body pair"))))
 
     (λ on-compile-error [_ msg ast call-me-to-reset-the-compiler]
-      (let [range (or (message.ast->range ast file)
+      (let [range (or (message.ast->range self file ast)
-                      (message.pos->range 0 0 0 0))]
+                      (line+byte->range self file 1 1))]
         (table.insert diagnostics
           {:range range
            :message msg
@@ -224,10 +232,9 @@ later by fennel-ls.language to answer requests from the client."
           (call-me-to-reset-the-compiler)
           (error "__NOT_AN_ERROR"))))
 
-    (λ on-parse-error [msg file line byte]
+    (λ on-parse-error [msg filename line byte _source call-me-to-reset-the-compiler]
-      ;; assume byte and char count is the same, ie no UTF-8
+      (let [line (if (= line "?") 1 line)
-      (let [line (- line 1)
+            range (line+byte->range self file line byte)]
-            range (message.pos->range line byte line byte)]
         (table.insert diagnostics
           {:range range
            :message msg
@@ -236,7 +243,9 @@ later by fennel-ls.language to answer requests from the client."
            :codeDescription "parse error"}))
       (if (recoverable? msg)
         true
-        (error "__NOT_AN_ERROR")))
+        (do
+          (call-me-to-reset-the-compiler)
+          (error "__NOT_AN_ERROR"))))
 
     (local allowed-globals
       (icollect [k _ (pairs _G)]
@@ -247,7 +256,7 @@ later by fennel-ls.language to answer requests from the client."
     (let [macro-file? (= (: file.text :sub 1 24) ";; fennel-ls: macro-file")
           plugin
           {:name "fennel-ls"
-           :versions ["1.3.1"]
+           :versions ["1.3.2"]
            : symbol-to-expression
            : call
            : destructure
@@ -266,8 +275,20 @@ later by fennel-ls.language to answer requests from the client."
                 :allowedGlobals allowed-globals
                 :requireAsInclude false
                 : scope}
-          parser (partial pcall (fennel.parser file.text file.uri opts))
+
-          ast (icollect [ok ok-2 ast parser &until (not (and ok ok-2))] ast)]
+          parser (let [p (fennel.parser file.text file.uri opts)]
+                   (fn p1 [p2 p3]
+                     (case (xpcall #(p p2 p3) fennel.traceback)
+                       (true r1 r2) (values r1 r2)
+                       (where (or (nil err) (false err)) (not (err:find "^[^\n]-__NOT_AN_ERROR\n")))
+                       (if (os.getenv :TESTING)
+                         (error (.. "\nYou have crashed the fennel parser or fennel-ls with the following message\n:" err
+                                    "\n\n^^^ the error message above here is the root problem\n\n"))
+                         (table.insert diagnostics
+                           {:range (line+byte->range self file 1 1)
+                            :message (.. "unrecoverable compiler error: " err)})))))
+
+          ast (icollect [ok ast parser &until (not ok)] ast)]
 
 
       ;; This is bad; we mutate fennel.macro-path
@@ -282,7 +303,7 @@ later by fennel-ls.language to answer requests from the client."
               (error (.. "\nYou have crashed the fennel compiler or fennel-ls with the following message\n:" err
                          "\n\n^^^ the error message above here is the root problem\n\n"))
               (table.insert diagnostics
-                {:range (message.pos->range 0 0 0 0)
+                {:range (line+byte->range self file 1 1)
                  :message (.. "unrecoverable compiler error: " err)}))))
 
         (set fennel.macro-path old-macro-path))

src/fennel-ls/diagnostics.fnl

@@ -11,7 +11,7 @@ Goes through a file and mutates the `file.diagnostics` field, filling it with di
   (icollect [symbol definition (pairs file.definitions) &into file.diagnostics]
     (if (and (= 0 (length definition.referenced-by))
              (not= "_" (: (tostring symbol) :sub 1 1)))
-      {:range (message.ast->range symbol file)
+      {:range (message.ast->range self file symbol)
        :message (.. "unused definition: " (tostring symbol))
        :severity message.severity.WARN
        :code 301
@@ -24,7 +24,7 @@ Goes through a file and mutates the `file.diagnostics` field, filling it with di
       (let [opts {}
             item (language.search self file symbol [] opts)]
         (if (and (not item) opts.searched-through-require)
-          {:range (message.ast->range symbol file)
+          {:range (message.ast->range self file symbol)
            :message (.. "unknown field " (tostring symbol))
            :severity message.severity.WARN
            :code 302

src/fennel-ls/handlers.fnl

@@ -18,7 +18,7 @@ Every time the client sends a message, it gets handled by a function in the corr
 (local notifications [])
 
 (local capabilities
-  {:textDocumentSync 1 ;; FIXME: upgrade to 2
+  {:textDocumentSync {:openClose true :change 2}
    ;; :notebookDocumentSync nil
    :completionProvider {:workDoneProgress false} ;; TODO
    :hoverProvider {:workDoneProgress false
@@ -69,25 +69,24 @@ Every time the client sends a message, it gets handled by a function in the corr
 
 (λ requests.textDocument/definition [self send {: position :textDocument {: uri}}]
   (let [file (state.get-by-uri self uri)
-        byte (utils.pos->byte file.text position.line position.character)]
+        byte (utils.position->byte file.text position self.position-encoding)]
     (case-try (language.find-symbol file.ast byte)
-      (symbol parents)
+      (symbol [parent])
-      ;; TODO unruin this match-try
+      (if
-      (let [parent (. parents 1)]
+        ;; require call
-        (if (. file.require-calls parent)
+        (. file.require-calls parent)
-          (language.search self file parent [] {:stop-early? true})
+        (language.search self file parent [] {:stop-early? true})
-          (language.search-main self file symbol {:stop-early? true} byte)))
+        ;; regular symbol
+        (language.search-main self file symbol {:stop-early? true} byte))
       (result result-file)
-      (message.range-and-uri
+      (message.range-and-uri self result-file (or result.binding result.definition))
-        (or result.binding result.definition)
-        result-file)
       (catch _ nil))))
 
-(λ requests.textDocument/references [self send {:position {: line : character}
+(λ requests.textDocument/references [self send {: position
                                                 :textDocument {: uri}
                                                 :context {:includeDeclaration ?include-declaration?}}]
   (let [file (state.get-by-uri self uri)
-        byte (utils.pos->byte file.text line character)]
+        byte (utils.position->byte file.text position self.position-encoding)]
     (case-try (language.find-symbol file.ast byte)
       symbol
       (if (. file.definitions symbol)
@@ -97,10 +96,10 @@ Every time the client sends a message, it gets handled by a function in the corr
       (let [result
             (icollect [_ symbol (ipairs referenced-by)]
               ;; TODO I currently assume all references are in the same file
-              (message.range-and-uri symbol result-file))]
+              (message.range-and-uri self result-file symbol))]
         (if ?include-declaration?
           (table.insert result
-                (message.range-and-uri definition.binding result-file)))
+            (message.range-and-uri self result-file definition.binding)))
 
         ;; TODO don't include duplicates
         result)
@@ -108,11 +107,11 @@ Every time the client sends a message, it gets handled by a function in the corr
 
 (λ requests.textDocument/hover [self send {: position :textDocument {: uri}}]
   (let [file (state.get-by-uri self uri)
-        byte (utils.pos->byte file.text position.line position.character)]
+        byte (utils.position->byte file.text position self.position-encoding)]
     (case-try (language.find-symbol file.ast byte)
       symbol (language.search-main self file symbol {} byte)
       result {:contents (formatter.hover-format result)
-              :range (message.ast->range symbol file)}
+              :range (message.ast->range self file symbol)}
       (catch _ nil))))
 
 ;; All of the helper functions for textDocument/completion are here until I
@@ -177,7 +176,7 @@ Every time the client sends a message, it gets handled by a function in the corr
 
 (λ requests.textDocument/completion [self send {: position :textDocument {: uri}}]
   (let [file (state.get-by-uri self uri)
-        byte (utils.pos->byte file.text position.line position.character)
+        byte (utils.position->byte file.text position self.position-encoding)
         (?symbol parents) (language.find-symbol file.ast byte)]
     (case (-?> ?symbol utils.multi-sym-split)
       (where (or nil [_ nil])) (scope-completion self file byte ?symbol parents)
@@ -186,7 +185,7 @@ Every time the client sends a message, it gets handled by a function in the corr
 
 (λ notifications.textDocument/didChange [self send {: contentChanges :textDocument {: uri}}]
   (local file (state.get-by-uri self uri))
-  (state.set-uri-contents self uri (utils.apply-changes file.text contentChanges))
+  (state.set-uri-contents self uri (utils.apply-changes file.text contentChanges self.position-encoding))
   (diagnostics.check self file)
   (send (message.diagnostics file)))
 

src/fennel-ls/language.fnl

@@ -2,7 +2,7 @@
 The high level analysis system that does deep searches following
 the data provided by compiler.fnl."
 
-(local {: sym? : list? : sequence? : varg? : sym : view : list} (require :fennel))
+(local {: sym? : list? : sequence? : varg? : sym : view} (require :fennel))
 (local utils (require :fennel-ls.utils))
 (local state (require :fennel-ls.state))
 

src/fennel-ls/message.fnl

@@ -52,26 +52,18 @@ to look to fix this in the future."
    : id
    :result ?result})
 
-(λ pos->range [sl sc el ec]
+(λ ast->range [self file ?ast]
-  {:start {:line sl :character sc}
-   :end   {:line el :character ec}})
-
-(λ ast->range [?ast file]
   (case (values (utils.get-ast-info ?ast :bytestart)
                 (utils.get-ast-info ?ast :byteend))
-    (i j)
+    (bytestart byteend)
-    (let [(start-line start-col) (utils.byte->pos file.text i)
+    {:start (utils.byte->position file.text bytestart self.position-encoding)
-          (end-line   end-col)   (utils.byte->pos file.text (+ j 1))]
+     :end   (utils.byte->position file.text (+ byteend 1) self.position-encoding)}))
-      (pos->range start-line start-col end-line end-col))))
 
-(λ range-and-uri [?ast {: uri &as file}]
+(λ range-and-uri [self {: uri &as file} ?ast]
   "if possible, returns the location of a symbol"
-  (case (ast->range ?ast file)
+  (case (ast->range self file ?ast)
     range {: range : uri}))
 
-(λ log [msg]
-  (create-notification :window/logMessage {: msg :type 4}))
-
 (λ diagnostics [file]
   (create-notification
     "textDocument/publishDiagnostics"
@@ -82,9 +74,7 @@ to look to fix this in the future."
  : create-request
  : create-response
  : create-error
- : pos->range
  : ast->range
- : log
  : range-and-uri
  : diagnostics
  : severity}

src/fennel-ls/state.fnl

@@ -26,21 +26,21 @@ in the \"self\" object."
 
 (λ get-by-module [self module]
   ;; check the cache
-  (match (. self.modules module)
+  (case (. self.modules module)
     uri
     (or (get-by-uri self uri)
       ;; if the cached uri isn't found, clear the cache and try again
       (do (tset self.modules module nil)
           (get-by-module self module)))
     nil
-    (match (searcher.lookup self module)
+    (case (searcher.lookup self module)
       uri
       (do
         (tset self.modules module uri)
         (get-by-uri self uri)))))
 
 (λ set-uri-contents [self uri text]
-  (match (. self.files uri)
+  (case (. self.files uri)
     ;; modify existing file
     file
     (do
@@ -77,7 +77,7 @@ in the \"self\" object."
 (fn make-configuration-from-template [default ?user ?parent]
   (if (= option-mt (getmetatable default))
       (let [setting
-             (match-try ?user
+             (case-try ?user
                nil (?. ?parent :all)
                nil (. default 1))]
         (assert (= (type (. default 1)) (type setting)))

src/fennel-ls/utils-utf16-surrogate-pairs.fnl

@@ -1,50 +0,0 @@
-(fn utf [byte]
-  "returns the number of (utf8) bytes, and (utf-16) code units from the first byte of a character"
-  (if
-    (<= 0x00 byte 0x80)
-    (values 1 1)
-    (<= 0xC0 byte 0xDF)
-    (values 2 1)
-    (<= 0xE0 byte 0xEF)
-    (values 3 1)
-    (<= 0xF0 byte 0xF7)
-    (values 4 2)
-    (error :utf8-error)))
-
-(fn byte->unit16 [str ?byte]
-  "convert from normal units to utf16 garbage"
-  (let [unit8 (or ?byte (length str))]
-    (var o8 0)
-    (var o16 0)
-    (while (< o8 unit8)
-      (let [(a8 a16) (utf (str:byte (+ 1 o8)))]
-        (set o8 (+ o8 a8))
-        (set o16 (+ o16 a16))))
-    (if (= o8 unit8)
-      o16
-      (error :utf8-error))))
-
-
-(fn unit16->byte [str unit16]
-  "convert from utf16 garbage to normal units"
-  (var o8 0)
-  (var o16 0)
-  (while (< o16 unit16)
-    (let [(a8 a16) (utf (str:byte (+ 1 o8)))]
-      (set o8 (+ o8 a8))
-      (set o16 (+ o16 a16))))
-  (if (= o16 unit16)
-    o8
-    (error :utf8-error)))
-
-
-(print (byte->unit16 "aλb𐐀" 1) 1)
-(print (byte->unit16 "aλb𐐀" 3) 2)
-(print (byte->unit16 "aλb𐐀" 4) 3)
-(print (byte->unit16 "aλb𐐀") 5)
-
-(print (unit16->byte "aλb𐐀" 1) 1)
-(print (unit16->byte "aλb𐐀" 2) 3)
-(print (unit16->byte "aλb𐐀" 3) 4)
-(print (unit16->byte "aλb𐐀" 5) 8)
-

src/fennel-ls/utils.fnl

@@ -3,6 +3,90 @@ A collection of utility functions. Many of these convert data between a
 Language-Server-Protocol representation and a Lua representation.
 These functions are all pure functions, which makes me happy."
 
+(λ next-line [str ?from]
+  "Find the start of the next line from a given byte offset, or from the start of the string."
+  (let [from (or ?from 1)]
+    (case (str:find "[\r\n]" from)
+      i (+ i (length (str:match "\r?\n?" i)))
+      nil nil)))
+
+(λ next-lines [str nlines ?from]
+  "Find the start of the next line from a given byte offset, or from the start of the string."
+  (faccumulate [from (or ?from 1)
+                i 1 nlines]
+    (next-line str from)))
+
+(fn utf [byte]
+  "returns the number of (utf8) bytes, and (utf-16) code units from the first byte of a character"
+  (if
+    (<= 0x00 byte 0x80)
+    (values 1 1)
+    (<= 0xC0 byte 0xDF)
+    (values 2 1)
+    (<= 0xE0 byte 0xEF)
+    (values 3 1)
+    (<= 0xF0 byte 0xF7)
+    (values 4 2)
+    (error :utf8-error)))
+
+(fn byte->unit16 [str ?byte]
+  "convert from normal units to utf16 garbage"
+  ;; TODO reconsider this when upstream #180 is fixed
+  (let [unit8 (math.min (length str) ?byte)]
+    (var o8 0)
+    (var o16 0)
+    (while (< o8 unit8)
+      (let [(a8 a16) (utf (str:byte (+ 1 o8)))]
+        (set o8 (+ o8 a8))
+        (set o16 (+ o16 a16))))
+    (if (= o8 unit8)
+      o16
+      (error :utf8-error))))
+
+
+(fn unit16->byte [str unit16]
+  "convert from utf16 garbage to normal units"
+  (var o8 0)
+  (var o16 0)
+  (while (< o16 unit16)
+    (let [(a8 a16) (utf (str:byte (+ 1 o8)))]
+      (set o8 (+ o8 a8))
+      (set o16 (+ o16 a16))))
+  (if (= o16 unit16)
+    o8
+    (error :utf8-error)))
+
+(λ pos->position [str line character encoding]
+  (case encoding
+    :utf-8 {: line : character}
+    :utf-16 (let [pos (next-lines str line)]
+              {: line
+               :character (byte->unit16 (str:sub pos) character)})
+    _ (error (.. "unknown encoding: " encoding))))
+
+(λ byte->position [str byte encoding]
+  "take a 1-indexed byte, and convert it to an LSP position based on the given encoding"
+  (var line 0)
+  (var pos 1)
+  (while (let [npos (next-line str pos)]
+           (when (and npos (<= npos byte))
+             (set pos npos)
+             (set line (+ line 1))
+             true)))
+  (case encoding
+    :utf-8 {: line :character (- byte pos)}
+    :utf-16 {: line :character (byte->unit16 (str:sub pos) (- byte pos))}
+    _ (error (.. "unknown encoding: " encoding))))
+
+(λ position->byte [str {: line : character} encoding]
+  "take an LSP position and convert it to a 1-indexed byte based on the given encoding"
+  (let [pos (next-lines str line)]
+    (assert pos :bad-pos)
+    (case encoding
+      :utf-8 (+ pos character)
+      :utf-16 (+ pos (unit16->byte (str:sub pos) character))
+      _ (error (.. "unknown encoding: " encoding)))))
+
 (λ startswith [str pre]
   (let [len (length pre)]
     (= (str:sub 1 len) pre)))
@@ -17,55 +101,24 @@ These functions are all pure functions, which makes me happy."
   "Prepents the \"file://\" prefix to a path to turn it into a uri"
   (.. "file://" path))
 
-(λ next-line [str ?from]
+(λ replace [text start-position end-position replacement encoding]
-  "Find the start of the next line from a given byte offset, or from the start of the string."
+  "Replaces a range of text with a replacement, using the protocol's definition of range."
-  (let [from (or ?from 1)]
+  (let [start (position->byte text start-position encoding)
-    (case (str:find "[\r\n]" from)
+        end   (position->byte text end-position   encoding)]
-      i (+ i (length (str:match "\r?\n?" i)))
-      nil nil)))
-
-(λ pos->byte [str line col]
-  "convert a 0-indexed line and column into a 1-indexed byte. Doesn't yet handle UTF8 UTF16 magic from the protocol"
-  (var sofar 1)
-  (for [_ 1 line &until (not sofar)]
-    (set sofar (next-line str sofar)))
-  (if sofar
-    (+ sofar col)
-    nil))
-
-(λ byte->pos [str byte]
-  "convert a 1-indexed byte into a 0-indexed line and column. Doesn't yet handle UTF8 UTF16 magic from the protocol"
-  (local up-to (str:sub 1 (- byte 1)))
-  (var lines 0)
-  (var pos 1)
-  (var prev nil)
-  (while (do (set prev pos)
-             (set pos (next-line up-to pos))
-             pos)
-    (set lines (+ 1 lines)))
-  (values lines (+ (length up-to) (- prev) 1)))
-
-(λ replace [text start-line start-col end-line end-col replacement]
-  "Replaces a range of text with a replacement, using the protocol's definition of range. Doesn't yet handle UTF8 UTF16 magic from the protocol"
-  (let [start (pos->byte text start-line start-col)
-        end   (pos->byte text end-line   end-col)]
     (..
       (text:sub 1 (- start 1))
       replacement
       (text:sub end))))
 
-(λ apply-changes [initial-text contentChanges]
+(λ apply-changes [initial-text changes encoding]
-  "Takes a list of Language-Server-Protocol `contentChanges` and applies them to a piece of text. Doesn't yet handle UTF8 UTF16 magic from the protocol"
+  "Takes a list of Language-Server-Protocol `contentChanges` and applies them to a piece of text."
   (accumulate
     [contents initial-text
-     _ change (ipairs contentChanges)]
+     _ change (ipairs changes)]
     (case change
       ;; Handle a change
       {:range {: start : end} : text}
-      (replace contents
+      (replace contents start end text encoding)
-        start.line start.character
-        end.line   end.character
-        text)
       ;; A replacment of the entire body
       {: text}
       text)))
@@ -94,8 +147,9 @@ These functions are all pure functions, which makes me happy."
 
 {: uri->path
  : path->uri
- : pos->byte
+ : pos->position
- : byte->pos
+ : byte->position
+ : position->byte
  : apply-changes
  : multi-sym-split
  : get-ast-info

test/completion-test.fnl

@@ -57,7 +57,7 @@
 
   (describe "When the program doesn't compile"
     (it "still completes without requiring the close parentheses"
-      (check-completion "(fn foo [z]\n  (let [x 10 y 20]\n    " 1 2 [:x :y :z]))
+      (check-completion "(fn foo [z]\n  (let [x 10 y 20]\n    " 2 4 [:x :y :z]))
 
     (it "still completes with no body in the `let`"
       (check-completion "(let [x 10 y 20]\n  )" 1 2 [:x :y]))

test/init-macros.fnl

@@ -28,7 +28,7 @@
   `(match ,item
     ,pattern nil
     ?otherwise#
-    (error
+    (is false
       (.. "Pattern did not match:\n"
           (let [fennel# (require :fennel)]
             (fennel#.view ?otherwise#))

test/lust.lua

@@ -206,7 +206,7 @@ function lust.expect(v)
           err = nerr or err
         end
         if not res then
-          error(err or 'unknown failure', 2)
+          error(err or 'unknown failure')
         end
       end
     end

test/references-test.fnl

@@ -1,6 +1,5 @@
 (import-macros {: is-matching : describe : it : before-each} :test)
 (local is (require :test.is))
-(local message (require :fennel-ls.message))
 
 (local {: view} (require :fennel))
 (local {: ROOT-URI
@@ -8,6 +7,10 @@
 
 (local filename (.. ROOT-URI "/imaginary-file.fnl"))
 
+(fn range [a b c d]
+  {:start {:line a :character b}
+   :end   {:line c :character d}})
+
 (fn check-references [body line col ?expected]
   (let [client (doto (create-client)
                  (: :open-file! filename body))
@@ -20,22 +23,22 @@
 (describe "references"
   (it "finds a reference from let"
     (check-references "(let [x 10] x)" 0 12
-      [{:uri filename :range (message.pos->range 0 12 0 13)}]))
+      [{:uri filename :range (range 0 12 0 13)}]))
 
   (it "finds a reference from let"
     (check-references "(let [x 10] x)" 0 6
-      [{:uri filename :range (message.pos->range 0 12 0 13)}]))
+      [{:uri filename :range (range 0 12 0 13)}]))
 
   (let [x 10] x x x)
   (it "finds multiple reference from let"
     (check-references "(let [x 10] x x x)" 0 6
-      [{:uri filename :range (message.pos->range 0 12 0 13)}
+      [{:uri filename :range (range 0 12 0 13)}
-       {:uri filename :range (message.pos->range 0 14 0 15)}
+       {:uri filename :range (range 0 14 0 15)}
-       {:uri filename :range (message.pos->range 0 16 0 17)}]))
+       {:uri filename :range (range 0 16 0 17)}]))
 
   (it "finds a reference from fn"
     (check-references "(fn x []) x" 0 10
-      [{:uri filename :range (message.pos->range 0 10 0 11)}]))
+      [{:uri filename :range (range 0 10 0 11)}]))
 
   ; (it "finds a reference from fn"
   ;   (check-references "(fn x []) x" 0 4

test/string-processing-test.fnl

@@ -6,31 +6,67 @@
 
 (describe "utils"
 
-    ;; fixme:
+  (fn position [line character]
-    ;; test for errors on out of bounds
+    {: line : character})
-    ;; test for multiline edits
+
-    ;; test for unicode utf8 utf16 nightmare
+  (fn range [start-line start-col end-line end-col]
+    {:start (position start-line start-col) :end (position end-line   end-col)})
+
+  (it "converts position->byte properly"
+    (is.equal 1 (utils.position->byte "a𐐀λ\nbλ𐐀" (position 0 0) :utf-8))
+    (is.equal 2 (utils.position->byte "a𐐀λ\nbλ𐐀" (position 0 1) :utf-8))
+    (is.equal 6 (utils.position->byte "a𐐀λ\nbλ𐐀" (position 0 5) :utf-8))
+    (is.equal 8 (utils.position->byte "a𐐀λ\nbλ𐐀" (position 0 7) :utf-8))
+    (is.equal 9 (utils.position->byte "a𐐀λ\nbλ𐐀" (position 1 0) :utf-8))
+    (is.equal 10 (utils.position->byte "a𐐀λ\nbλ𐐀" (position 1 1) :utf-8))
+    (is.equal 12 (utils.position->byte "a𐐀λ\nbλ𐐀" (position 1 3) :utf-8))
+    (is.equal 16 (utils.position->byte "a𐐀λ\nbλ𐐀" (position 1 7) :utf-8))
+    (is.equal 1 (utils.position->byte "a𐐀λ\nbλ𐐀" (position 0 0) :utf-16))
+    (is.equal 2 (utils.position->byte "a𐐀λ\nbλ𐐀" (position 0 1) :utf-16))
+    (is.equal 6 (utils.position->byte "a𐐀λ\nbλ𐐀" (position 0 3) :utf-16))
+    (is.equal 8 (utils.position->byte "a𐐀λ\nbλ𐐀" (position 0 4) :utf-16))
+    (is.equal 9 (utils.position->byte "a𐐀λ\nbλ𐐀" (position 1 0) :utf-16))
+    (is.equal 10 (utils.position->byte "a𐐀λ\nbλ𐐀" (position 1 1) :utf-16))
+    (is.equal 12 (utils.position->byte "a𐐀λ\nbλ𐐀" (position 1 2) :utf-16))
+    (is.equal 16 (utils.position->byte "a𐐀λ\nbλ𐐀" (position 1 4) :utf-16)))
+
+  (it "converts byte->position properly"
+    (is.same (position 0 0) (utils.byte->position "a𐐀λ\nbλ𐐀" 1 :utf-8))
+    (is.same (position 0 1) (utils.byte->position "a𐐀λ\nbλ𐐀" 2 :utf-8))
+    (is.same (position 0 5) (utils.byte->position "a𐐀λ\nbλ𐐀" 6 :utf-8))
+    (is.same (position 0 7) (utils.byte->position "a𐐀λ\nbλ𐐀" 8 :utf-8))
+    (is.same (position 1 0) (utils.byte->position "a𐐀λ\nbλ𐐀" 9 :utf-8))
+    (is.same (position 1 1) (utils.byte->position "a𐐀λ\nbλ𐐀" 10 :utf-8))
+    (is.same (position 1 3) (utils.byte->position "a𐐀λ\nbλ𐐀" 12 :utf-8))
+    (is.same (position 1 7) (utils.byte->position "a𐐀λ\nbλ𐐀" 16 :utf-8))
+    (is.same (position 0 0) (utils.byte->position "a𐐀λ\nbλ𐐀" 1 :utf-16))
+    (is.same (position 0 1) (utils.byte->position "a𐐀λ\nbλ𐐀" 2 :utf-16))
+    (is.same (position 0 3) (utils.byte->position "a𐐀λ\nbλ𐐀" 6 :utf-16))
+    (is.same (position 0 4) (utils.byte->position "a𐐀λ\nbλ𐐀" 8 :utf-16))
+    (is.same (position 1 0) (utils.byte->position "a𐐀λ\nbλ𐐀" 9 :utf-16))
+    (is.same (position 1 1) (utils.byte->position "a𐐀λ\nbλ𐐀" 10 :utf-16))
+    (is.same (position 1 2) (utils.byte->position "a𐐀λ\nbλ𐐀" 12 :utf-16))
+    (is.same (position 1 4) (utils.byte->position "a𐐀λ\nbλ𐐀" 16 :utf-16)))
 
   (describe "apply-changes"
 
-    (fn range [start-line start-col end-line end-col]
-      {:start {:line start-line :character start-col}
-       :end   {:line end-line   :character end-col}})
-
     (it "updates the start of a line"
       (is.equal
         (utils.apply-changes
           "replace beginning"
           [{:range (range 0 0 0 7)
-            :text "the"}])
+            :text "the"}]
+          :utf-8)
         "the beginning"))
 
+
     (it "updates the end of a line"
       (is.equal
         (utils.apply-changes
           "first line\nsecond line\nreplace end"
           [{:range (range 2 7 2 11)
-            :text "ment"}])
+            :text "ment"}]
+          :utf-8)
         "first line\nsecond line\nreplacement"))
 
     (it "replaces a line"
@@ -38,23 +74,25 @@
         (utils.apply-changes
           "replace all"
           [{:range (range 0 0 0 11)
-            :text "new string"}])
+            :text "new string"}]
+          :utf-8)
         "new string"))
 
     (it "can handle substituting things"
       (is.equal
         (utils.apply-changes
           "replace beginning"
-          [{:range {:start {:line 0 :character 0}
+          [{:range (range 0 0 0 7)
-                    :end   {:line 0 :character 7}}
+            :text "the"}]
-            :text "the"}])
+          :utf-8)
         "the beginning"))
 
     (it "can handle replacing everything"
       (is.equal
         (utils.apply-changes
           "this is the\nold file"
-          [{:text "And this is the\nnew file"}])
+          [{:text "And this is the\nnew file"}]
+          :utf-8)
         "And this is the\nnew file"))))
 
  ;; (it "can substitute multiple ranges")