diff --git a/test/string-processing-test.fnl b/test/string-processing-test.fnl index 7a49009..b4c6c84 100644 --- a/test/string-processing-test.fnl +++ b/test/string-processing-test.fnl @@ -12,6 +12,13 @@ (fn range [start-line start-col end-line end-col] {:start (position start-line start-col) :end (position end-line end-col)}) + ;; "a" U+0061 is in U+0000 to U+007F, and therefore is 1 byte in UTF-8, and 1 codepoint in UTF-16 + ;; "λ" U+03BB is in U+0080 to U+07FF, and therefore is 2 bytes in UTF-8, and 1 codepoint in UTF-16 + ;; "セ" U+FF7E is in U+0800 to U+FFFF, and therefore is 3 bytes in UTF-8, and 1 codepoint in UTF-16 + ;; "𐐀" U+10400 is in U+10000 to U+10FFFF,and therefore is 4 bytes in UTF-8, and 2 codepoints in UTF-16 + ;; These symbols cover each of the four cases of byte/codepoint widths + ;; they should be sufficient for testing + (it "converts position->byte properly" (is.equal 1 (utils.position->byte "a𐐀λ\nbλ𐐀" (position 0 0) :utf-8)) (is.equal 2 (utils.position->byte "a𐐀λ\nbλ𐐀" (position 0 1) :utf-8)) @@ -28,7 +35,10 @@ (is.equal 9 (utils.position->byte "a𐐀λ\nbλ𐐀" (position 1 0) :utf-16)) (is.equal 10 (utils.position->byte "a𐐀λ\nbλ𐐀" (position 1 1) :utf-16)) (is.equal 12 (utils.position->byte "a𐐀λ\nbλ𐐀" (position 1 2) :utf-16)) - (is.equal 16 (utils.position->byte "a𐐀λ\nbλ𐐀" (position 1 4) :utf-16))) + (is.equal 16 (utils.position->byte "a𐐀λ\nbλ𐐀" (position 1 4) :utf-16)) + (is.equal 19 (utils.position->byte "a𐐀セλ\nbλ𐐀" (position 1 4) :utf-16)) + (is.equal 19 (utils.position->byte "a𐐀セλ\nbλ𐐀" (position 1 4) :utf-16)) + (is.equal 7 (utils.position->byte "セセ" (position 0 2) :utf-16))) (it "converts byte->position properly" (is.same (position 0 0) (utils.byte->position "a𐐀λ\nbλ𐐀" 1 :utf-8)) @@ -46,7 +56,10 @@ (is.same (position 1 0) (utils.byte->position "a𐐀λ\nbλ𐐀" 9 :utf-16)) (is.same (position 1 1) (utils.byte->position "a𐐀λ\nbλ𐐀" 10 :utf-16)) (is.same (position 1 2) (utils.byte->position "a𐐀λ\nbλ𐐀" 12 :utf-16)) - (is.same (position 1 4) (utils.byte->position "a𐐀λ\nbλ𐐀" 16 :utf-16))) + (is.same (position 1 4) (utils.byte->position "a𐐀λ\nbλ𐐀" 16 :utf-16)) + (is.same (position 1 4) (utils.byte->position "a𐐀セλ\nbλ𐐀" 19 :utf-16)) + (is.same (position 1 4) (utils.byte->position "a𐐀セλ\nbλ𐐀" 19 :utf-16)) + (is.same (position 0 2) (utils.byte->position "セセ" 7 :utf-16))) (describe "apply-changes"