From b6b51698c4d82a8d502fa21f5745e227eb8d9e3f Mon Sep 17 00:00:00 2001 From: XeroOl Date: Sat, 3 Feb 2024 01:32:18 -0600 Subject: [PATCH] =?UTF-8?q?added=20"=EF=BD=BE"=20character=20to=20unicode?= =?UTF-8?q?=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Although it was properly handled it before, there were no tests for the unicode offset conversions that used characters that are 3 bytes in utf8 and 2 bytes in utf16, so I added one. Now, every single ""type"" of charater is tested. --- test/string-processing-test.fnl | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/test/string-processing-test.fnl b/test/string-processing-test.fnl index 7a49009..b4c6c84 100644 --- a/test/string-processing-test.fnl +++ b/test/string-processing-test.fnl @@ -12,6 +12,13 @@ (fn range [start-line start-col end-line end-col] {:start (position start-line start-col) :end (position end-line end-col)}) + ;; "a" U+0061 is in U+0000 to U+007F, and therefore is 1 byte in UTF-8, and 1 codepoint in UTF-16 + ;; "λ" U+03BB is in U+0080 to U+07FF, and therefore is 2 bytes in UTF-8, and 1 codepoint in UTF-16 + ;; "ｾ" U+FF7E is in U+0800 to U+FFFF, and therefore is 3 bytes in UTF-8, and 1 codepoint in UTF-16 + ;; "𐐀" U+10400 is in U+10000 to U+10FFFF,and therefore is 4 bytes in UTF-8, and 2 codepoints in UTF-16 + ;; These symbols cover each of the four cases of byte/codepoint widths + ;; they should be sufficient for testing + (it "converts position->byte properly" (is.equal 1 (utils.position->byte "a𐐀λ\nbλ𐐀" (position 0 0) :utf-8)) (is.equal 2 (utils.position->byte "a𐐀λ\nbλ𐐀" (position 0 1) :utf-8)) @@ -28,7 +35,10 @@ (is.equal 9 (utils.position->byte "a𐐀λ\nbλ𐐀" (position 1 0) :utf-16)) (is.equal 10 (utils.position->byte "a𐐀λ\nbλ𐐀" (position 1 1) :utf-16)) (is.equal 12 (utils.position->byte "a𐐀λ\nbλ𐐀" (position 1 2) :utf-16)) - (is.equal 16 (utils.position->byte "a𐐀λ\nbλ𐐀" (position 1 4) :utf-16))) + (is.equal 16 (utils.position->byte "a𐐀λ\nbλ𐐀" (position 1 4) :utf-16)) + (is.equal 19 (utils.position->byte "a𐐀ｾλ\nbλ𐐀" (position 1 4) :utf-16)) + (is.equal 19 (utils.position->byte "a𐐀ｾλ\nbλ𐐀" (position 1 4) :utf-16)) + (is.equal 7 (utils.position->byte "ｾｾ" (position 0 2) :utf-16))) (it "converts byte->position properly" (is.same (position 0 0) (utils.byte->position "a𐐀λ\nbλ𐐀" 1 :utf-8)) @@ -46,7 +56,10 @@ (is.same (position 1 0) (utils.byte->position "a𐐀λ\nbλ𐐀" 9 :utf-16)) (is.same (position 1 1) (utils.byte->position "a𐐀λ\nbλ𐐀" 10 :utf-16)) (is.same (position 1 2) (utils.byte->position "a𐐀λ\nbλ𐐀" 12 :utf-16)) - (is.same (position 1 4) (utils.byte->position "a𐐀λ\nbλ𐐀" 16 :utf-16))) + (is.same (position 1 4) (utils.byte->position "a𐐀λ\nbλ𐐀" 16 :utf-16)) + (is.same (position 1 4) (utils.byte->position "a𐐀ｾλ\nbλ𐐀" 19 :utf-16)) + (is.same (position 1 4) (utils.byte->position "a𐐀ｾλ\nbλ𐐀" 19 :utf-16)) + (is.same (position 0 2) (utils.byte->position "ｾｾ" 7 :utf-16))) (describe "apply-changes"