The parser is more robust under error

README.md

@@ -25,27 +25,15 @@ Features / To Do List / Things I would enjoy patches for:
     - [ ] goes to a.method on `(: a :method)` when triggered at `:method`
     - [X] expanded macros (a little bit)
     - [ ] table mutation via `fn` special: `(fn obj.new-field [])`
-    - [ ] local/table mutation via set/tset
     - [ ] macro calls / which macros are in scope
-    - [ ] .lua files (antifennel decompiler)
     - [ ] setmetatable
     - [ ] function arguments / function calls
+    - [ ] local/table mutation via set/tset
+    - [ ] .lua files (antifennel decompiler)
     - [ ] mutation on aliased tables (difficult)
-- [X] Reports compiler errors
-- [ ] Reports linting issues
-    - [ ] Unused locals
-    - [ ] Discarding results from pcall/xpcall/other functions
-    - [ ] `unpack` or `values` into an operator special
-    - [ ] `do`/`values` with only one inner form
-    - [ ] redundant `do` as the last/only item in a form that accepts a "body"
-    - [ ] `var` forms that could be `local`
-    - [ ] Dead code (I'm not sure what sort of things cause dead code)
-    - [ ] Unused fields (difficult)
-    - [ ] unification in a `match` pattern (difficult)
-    - [ ] Brainstorm more linting patterns (I spent a couple minutes brainstorming these ideas, other ideas are welcome of course)
 - [ ] Completion Suggestions
     - [X] from globals
-    - [ ] from current scope
+    - [X] from current scope
     - [ ] from macros (only on first form in a list)
     - [ ] from specials (only on first form in a list)
     - [ ] "dot completion" for table fields
@@ -58,6 +46,19 @@ Features / To Do List / Things I would enjoy patches for:
     - [ ] `(: "foo" :` string completions
     - [ ] `(require :` module completions
     - [ ] snippets? I guess?
+- [X] Reports compiler errors
+    - [.] Report more than one error per top-level form
+- [ ] Reports linting issues
+    - [ ] Unused locals
+    - [ ] Discarding results from pcall/xpcall/other functions
+    - [ ] `unpack` or `values` into an operator special
+    - [ ] `do`/`values` with only one inner form
+    - [ ] redundant `do` as the last/only item in a form that accepts a "body"
+    - [ ] `var` forms that could be `local`
+    - [ ] Dead code (I'm not sure what sort of things cause dead code)
+    - [ ] Unused fields (difficult)
+    - [ ] unification in a `match` pattern (difficult)
+    - [ ] Brainstorm more linting patterns (I spent a couple minutes brainstorming these ideas, other ideas are welcome of course)
 - [X] Hover over a symbol for documentation
 - [ ] Signature help
     - [ ] Regular help
@@ -75,9 +76,10 @@ Features / To Do List / Things I would enjoy patches for:
     - [ ] lua version
     - [ ] allowed global list
     - [ ] enable/disable various linters
-- [ ] rename local symbols
+- [ ] rename
-- [ ] rename module fields (may affect code behavior, may modify other files)
+    - [ ] local symbols
-- [ ] rename arbitrary things (may affect code behavior, may modify other files)
+    - [ ] module fields (may affect code behavior, may modify other files)
+    - [ ] arbitrary fields (may affect code behavior, may modify other files)
 - [ ] formatting with fnlfmt
 - [ ] Type annotations? Global type inference?
 
@@ -90,10 +92,11 @@ make
 ```
 
 2. Configure your editor to use this language server
-LSP is editor-agnostic, but that's only if you're able to actually follow the spec, and I'm not sure that fennel-ls is compliant.
+LSP is editor-agnostic, but that's only if you're able to actually follow the spec, and I'm not sure if fennel-ls is compliant.
 
 So far, I've only ever tested it with Neovim using the native language client and `lspconfig`.
 If you know what that means, here's the relevant code to help you set up Neovim in the same way:
+
 ```lua
 local lspconfig = require('lspconfig')
 -- inform lspconfig about fennel-ls

fennel

@@ -3876,47 +3876,6 @@ package.preload["fennel.parser"] = package.preload["fennel.parser"] or function(
           return nil
         end
       end
-      local function badend()
-        local accum = utils.map(stack, "closer")
-        local _218_
-        if (#stack == 1) then
-          _218_ = ""
-        else
-          _218_ = "s"
-        end
-        return parse_error(string.format("expected closing delimiter%s %s", _218_, string.char(unpack(accum))))
-      end
-      local function skip_whitespace(b)
-        if (b and whitespace_3f(b)) then
-          whitespace_since_dispatch = true
-          return skip_whitespace(getb())
-        elseif (not b and (0 < #stack)) then
-          return badend()
-        else
-          return b
-        end
-      end
-      local function parse_comment(b, contents)
-        if (b and (10 ~= b)) then
-          local function _222_()
-            local _221_ = contents
-            table.insert(_221_, string.char(b))
-            return _221_
-          end
-          return parse_comment(getb(), _222_())
-        elseif comments then
-          return dispatch(utils.comment(table.concat(contents), {line = (line - 1), filename = filename}))
-        else
-          return b
-        end
-      end
-      local function open_table(b)
-        if not whitespace_since_dispatch then
-          parse_error(("expected whitespace before opening delimiter " .. string.char(b)))
-        else
-        end
-        return table.insert(stack, {bytestart = byteindex, closer = delims[b], filename = filename, line = line, col = (col - 1)})
-      end
       local function close_list(list)
         return dispatch(setmetatable(list, getmetatable(utils.list())))
       end
@@ -3928,12 +3887,12 @@ package.preload["fennel.parser"] = package.preload["fennel.parser"] or function(
         return dispatch(val)
       end
       local function add_comment_at(comments0, index, node)
-        local _225_ = (comments0)[index]
+        local _218_ = (comments0)[index]
-        if (nil ~= _225_) then
+        if (nil ~= _218_) then
-          local existing = _225_
+          local existing = _218_
           return table.insert(existing, node)
         elseif true then
-          local _ = _225_
+          local _ = _218_
           comments0[index] = {node}
           return nil
         else
@@ -4011,6 +3970,51 @@ package.preload["fennel.parser"] = package.preload["fennel.parser"] or function(
           return close_curly_table(top)
         end
       end
+      local function badend(cause)
+        local accum = utils.map(stack, "closer")
+        local _228_
+        if (#stack == 1) then
+          _228_ = ""
+        else
+          _228_ = "s"
+        end
+        parse_error(string.format("expected closing delimiter%s %s", _228_, string.char(unpack(accum))))
+        for i = #accum, 2, -1 do
+          close_table(accum[i])
+        end
+        return accum[1]
+      end
+      local function skip_whitespace(b)
+        if (b and whitespace_3f(b)) then
+          whitespace_since_dispatch = true
+          return skip_whitespace(getb())
+        elseif (not b and (0 < #stack)) then
+          return badend("eof")
+        else
+          return b
+        end
+      end
+      local function parse_comment(b, contents)
+        if (b and (10 ~= b)) then
+          local function _232_()
+            local _231_ = contents
+            table.insert(_231_, string.char(b))
+            return _231_
+          end
+          return parse_comment(getb(), _232_())
+        elseif comments then
+          return dispatch(utils.comment(table.concat(contents), {line = (line - 1), filename = filename}))
+        else
+          return b
+        end
+      end
+      local function open_table(b)
+        if not whitespace_since_dispatch then
+          parse_error(("expected whitespace before opening delimiter " .. string.char(b)))
+        else
+        end
+        return table.insert(stack, {bytestart = byteindex, closer = delims[b], filename = filename, line = line, col = (col - 1)})
+      end
       local function parse_string_loop(chars, b, state)
         table.insert(chars, b)
         local state0
@@ -4043,7 +4047,7 @@ package.preload["fennel.parser"] = package.preload["fennel.parser"] or function(
         table.insert(stack, {closer = 34})
         local chars = {34}
         if not parse_string_loop(chars, getb(), "base") then
-          badend()
+          badend("string")
         else
         end
         table.remove(stack)

src/fennel-ls.fnl

@@ -1,11 +1,16 @@
 (local dispatch (require :fennel-ls.dispatch))
 (local json-rpc (require :fennel-ls.json-rpc))
 
+(local log (io.open "/home/xerool/Documents/projects/fennel-ls/log.txt" "w"))
+(local {: view} (require :fennel))
+
 (λ main-loop [in out]
   (local send (partial json-rpc.write out))
   (local state [])
   (while true
     (let [msg (json-rpc.read in)]
+      (log:write (view msg) "\n")
+      (log:flush)
       (dispatch.handle state send msg))))
 
 (λ main []

src/fennel-ls/compiler.fnl

@@ -124,6 +124,13 @@ later by fennel-ls.language to answer requests from the client."
         [-require- modname]
         (tset require-calls ast true)))
 
+    (λ recoverable? [msg]
+      (or (msg:find "unknown identifier in strict mode")
+          (msg:find "expected closing delimiter")
+          (msg:find "expected body expression")
+          (msg:find "expected whitespace before opening delimiter")
+          (msg:find "malformed multisym")))
+
     (λ on-compile-error [_ msg ast call-me-to-reset-the-compiler]
       (let [range (or (message.ast->range ast file)
                       (message.pos->range 0 0 0 0))]
@@ -133,8 +140,11 @@ later by fennel-ls.language to answer requests from the client."
            :severity message.severity.ERROR
            :code 201
            :codeDescription "compiler error"}))
+      (if (recoverable? msg)
+        true
+        (do
           (call-me-to-reset-the-compiler)
-      (error "__NOT_AN_ERROR"))
+          (error "__NOT_AN_ERROR"))))
 
     (λ on-parse-error [msg file line byte]
       ;; assume byte and char count is the same, ie no UTF-8
@@ -143,11 +153,16 @@ later by fennel-ls.language to answer requests from the client."
           {:range range
            :message msg
            :severity message.severity.ERROR
-           :code 201
+           :code 101
-           :codeDescription "compiler error"}))
+           :codeDescription "parse error"}))
-      (error "__NOT_AN_ERROR"))
+      (if (recoverable? msg)
+        true
+        (do
+          (print msg)
+          (error "__NOT_AN_ERROR"))))
 
     (local allowed-globals (icollect [k v (pairs _G)] k))
+    (table.insert allowed-globals :vim)
 
     ;; TODO clean up this code. It's awful now that there is error handling
     (let

src/fennel.lua

@@ -408,10 +408,10 @@ package.preload["fennel.repl"] = package.preload["fennel.repl"] or function(...)
           _676_ = _677_
         end
       end
-      if ((_G.type(_676_) == "table") and (nil ~= (_676_).source) and ((_676_).what == "Lua") and (nil ~= (_676_).short_src) and (nil ~= (_676_).linedefined)) then
+      if ((_G.type(_676_) == "table") and ((_676_).what == "Lua") and (nil ~= (_676_).linedefined) and (nil ~= (_676_).source) and (nil ~= (_676_).short_src)) then
+        local line = (_676_).linedefined
         local source = (_676_).source
         local src = (_676_).short_src
-        local line = (_676_).linedefined
         local fnlsrc
         do
           local t_681_ = compiler.sourcemap
@@ -3662,47 +3662,6 @@ package.preload["fennel.parser"] = package.preload["fennel.parser"] or function(
           return nil
         end
       end
-      local function badend()
-        local accum = utils.map(stack, "closer")
-        local _218_
-        if (#stack == 1) then
-          _218_ = ""
-        else
-          _218_ = "s"
-        end
-        return parse_error(string.format("expected closing delimiter%s %s", _218_, string.char(unpack(accum))))
-      end
-      local function skip_whitespace(b)
-        if (b and whitespace_3f(b)) then
-          whitespace_since_dispatch = true
-          return skip_whitespace(getb())
-        elseif (not b and (0 < #stack)) then
-          return badend()
-        else
-          return b
-        end
-      end
-      local function parse_comment(b, contents)
-        if (b and (10 ~= b)) then
-          local function _222_()
-            local _221_ = contents
-            table.insert(_221_, string.char(b))
-            return _221_
-          end
-          return parse_comment(getb(), _222_())
-        elseif comments then
-          return dispatch(utils.comment(table.concat(contents), {line = (line - 1), filename = filename}))
-        else
-          return b
-        end
-      end
-      local function open_table(b)
-        if not whitespace_since_dispatch then
-          parse_error(("expected whitespace before opening delimiter " .. string.char(b)))
-        else
-        end
-        return table.insert(stack, {bytestart = byteindex, closer = delims[b], filename = filename, line = line, col = (col - 1)})
-      end
       local function close_list(list)
         return dispatch(setmetatable(list, getmetatable(utils.list())))
       end
@@ -3714,12 +3673,12 @@ package.preload["fennel.parser"] = package.preload["fennel.parser"] or function(
         return dispatch(val)
       end
       local function add_comment_at(comments0, index, node)
-        local _225_ = (comments0)[index]
+        local _218_ = (comments0)[index]
-        if (nil ~= _225_) then
+        if (nil ~= _218_) then
-          local existing = _225_
+          local existing = _218_
           return table.insert(existing, node)
         elseif true then
-          local _ = _225_
+          local _ = _218_
           comments0[index] = {node}
           return nil
         else
@@ -3797,6 +3756,51 @@ package.preload["fennel.parser"] = package.preload["fennel.parser"] or function(
           return close_curly_table(top)
         end
       end
+      local function badend(cause)
+        local accum = utils.map(stack, "closer")
+        local _228_
+        if (#stack == 1) then
+          _228_ = ""
+        else
+          _228_ = "s"
+        end
+        parse_error(string.format("expected closing delimiter%s %s", _228_, string.char(unpack(accum))))
+        for i = #accum, 2, -1 do
+          close_table(accum[i])
+        end
+        return accum[1]
+      end
+      local function skip_whitespace(b)
+        if (b and whitespace_3f(b)) then
+          whitespace_since_dispatch = true
+          return skip_whitespace(getb())
+        elseif (not b and (0 < #stack)) then
+          return badend("eof")
+        else
+          return b
+        end
+      end
+      local function parse_comment(b, contents)
+        if (b and (10 ~= b)) then
+          local function _232_()
+            local _231_ = contents
+            table.insert(_231_, string.char(b))
+            return _231_
+          end
+          return parse_comment(getb(), _232_())
+        elseif comments then
+          return dispatch(utils.comment(table.concat(contents), {line = (line - 1), filename = filename}))
+        else
+          return b
+        end
+      end
+      local function open_table(b)
+        if not whitespace_since_dispatch then
+          parse_error(("expected whitespace before opening delimiter " .. string.char(b)))
+        else
+        end
+        return table.insert(stack, {bytestart = byteindex, closer = delims[b], filename = filename, line = line, col = (col - 1)})
+      end
       local function parse_string_loop(chars, b, state)
         table.insert(chars, b)
         local state0
@@ -3829,7 +3833,7 @@ package.preload["fennel.parser"] = package.preload["fennel.parser"] or function(
         table.insert(stack, {closer = 34})
         local chars = {34}
         if not parse_string_loop(chars, getb(), "base") then
-          badend()
+          badend("string")
         else
         end
         table.remove(stack)

test/completion-test.fnl

@@ -10,7 +10,7 @@
 (local dispatch (require :fennel-ls.dispatch))
 (local message (require :fennel-ls.message))
 
-(local filename (.. ROOT-URI "imaginary-file.fnl"))
+(local filename (.. ROOT-URI "/imaginary-file.fnl"))
 
 (fn check-completion [body line col expected ?unexpected]
   (local state (doto [] setup-server))
@@ -18,8 +18,9 @@
   (let [response (dispatch.handle* state (completion-at filename line col))
         seen (collect [_ suggestion (ipairs (. response 1 :result))]
                 suggestion.label suggestion.label)]
+    (if (= (type expected) :table)
       (each [_ exp (ipairs expected)]
-      (is.truthy (. seen exp) (.. exp " was not suggested, but should be")))
+        (is.truthy (. seen exp) (.. exp " was not suggested, but should be"))))
     (if ?unexpected
       (each [_ exp (ipairs ?unexpected)]
         (is.nil (. seen exp) (.. exp " was suggested, but shouldn't be"))))))
@@ -42,17 +43,33 @@
     (check-completion "(d)" 0 3 [:do :doto]))
 
   (it "suggests macros in scope"
-    (check-completion "(macro funny [] `nil)\n()" 1 1 [:funny])))
+    (check-completion "(macro funny [] `nil)\n()" 1 1 [:funny]))
 
-  ;; ;; Compiler hardening
+  (it "does not suggest locals out of scope"
-  ;; (it "works without requiring the close parentheses"))
+    (check-completion "(do (local x 10))\n" 1 0 [] [:x]))
-  ;; (it "works without a body in the `let`"))
-  ;; (it "does not suggest locals out of scope")
-  ;; (it "suggests items from the previous definitions in the same `let`")
 
-  ;; ;; Functions
+  (it "does not suggest function args out of scope"
-  ;; (it "suggests function arguments at the top scope of the function")
+    (check-completion "(fn [x] (print x))\n" 1 0 [] [:x]))
-  ;; (it "suggests function arguments deep within the function")
+
+  (describe "when the program doesn't compile"
+    (it "still completes without requiring the close parentheses"
+      (check-completion "(fn foo [z]\n  (let [x 10 y 20]\n    " 1 2 [:x :y :z]))
+
+    (it "still completes with no body in the `let`"
+      (check-completion "(let [x 10 y 20]\n  )" 1 2 [:x :y]))
+
+    (it "still completes items from the previous definitions in the same `let`"
+      (check-completion "(let [a 10\n      b 20\n      " 1 6 [:a :b])))
+
+    ;; (it "completes fields with a partially typed multisym that ends in :"
+    ;;   (check-completion "(local x {:field (fn [])})\n(x:" 1 3 [:field])))
+
+  ;; Functions
+  (it "suggests function arguments at the top scope of the function"
+    (check-completion "(fn foo [arg1 arg2 arg3]\n  )" 1 2 [:arg1 :arg2 :arg3]))
+
+  (it "suggests function arguments at the top scope of the function"
+    (check-completion "(fn foo [arg1 arg2 arg3]\n  (do (do (do ))))" 1 14 [:arg1 :arg2 :arg3])))
 
   ;; ;; Scope Ordering Rules
   ;; (it "does not suggest locals past the suggestion location when a symbol is partially typed")
@@ -67,7 +84,6 @@
   ;;   (check-completion "(do )"
   ;;                     0 4 [] [:do :let :fn :-> :-?>> :?.])))
 
-
   ;; (it "doesn't suggest specials at the very top level")
   ;; (it "doesn't suggest macros in the middle of a list (open paren required)")
   ;; (it "doesn't suggest macros at the very top level")

test/diagnostic-test.fnl

@@ -20,7 +20,7 @@
   (table.insert t result)
   `(accumulate ,t ,body))
 
-(local filename (.. ROOT-URI "imaginary.fnl"))
+(local filename (.. ROOT-URI "/imaginary.fnl"))
 
 (describe "diagnostic messages"
   (it "handles compile errors"
@@ -59,7 +59,13 @@
           {:diagnostics
            [{:range {:start {:character a :line b}
                      :end   {:character c :line d}}}]}}]
-        "diagnostics should always have a range"))))
+        "diagnostics should always have a range")))
+
+  (it "gives more than one error"
+    (local state (doto [] setup-server))
+    (let [responses (open-file state filename "(unknown-global-1 unknown-global-2)")]
+      (is-matching responses
+        [{:params {:diagnostics [a b]}}]  "there should be a diagnostic for each one here"))))
 
 ;; TODO lints:
 ;; unnecessary (do) in body position

test/goto-definition-test.fnl

@@ -1,4 +1,5 @@
 (import-macros {: is-matching : describe : it : before-each} :test)
+
 (local is (require :luassert))
 
 (local {: ROOT-URI
@@ -72,6 +73,7 @@
     (check :goto-definition.fnl 47 13 :goto-definition.fnl 47 30 47 52)))
 
   ;; TODO
+  ;; (it "doesn't leak function arguments to the surrounding scope")
   ;; (it "can go to a function in another file imported via destructuring assignment") ;; WORKS, just needs a test case
   ;; (it "can go through more than one extra file")
   ;; (it "will give up instead of freezing on recursive requires")
@@ -79,11 +81,11 @@
   ;; (it "can follow import-macros (destructuring)")
   ;; (it "can follow import-macros (namespaced)")
   ;; (it "can go to the definition even in a lua file")
-  ;; (it "finds (fn a.b [] ...) declarations")
   ;; (it "finds (set a.b) definitions")
+  ;; (it "finds (fn a.b [] ...) declarations")
   ;; (it "finds (tset a :b) definitions")
   ;; (it "finds (setmetatable a {__index {:b def}) definitions")
-  ;; (it "finds definitions from inside a function (fn foo [] (local x 10) {: x}) (let [result (foo)] (print result.x)) finds result.x")
+  ;; (it "finds definitions into a function (fn foo [] (local x 10) {: x}) (let [result (foo)] (print result.x)) finds result.x")
   ;; (it "finds basic setmetatable definitions with an __index function")
   ;; (it "can return to callsite and go through a function's arguments when they're available")
   ;; (it "can go to a function's reference OR read type inference comments when callsite isn't available (PICK ONE)")

test/init-macros.fnl

@@ -2,18 +2,24 @@
 
 (fn it [desc ...]
   "busted's `it` function"
+  (let [body [...]]
+    (table.insert body `nil)
     `((. (require :busted) :it)
-    ,desc (fn [] ,desc ,...)))
+      ,desc (fn [] ,desc ,(unpack body)))))
 
 (fn describe [desc ...]
   "busted's `describe` function"
+  (let [body [...]]
+    (table.insert body `nil)
     `((. (require :busted) :describe)
-    ,desc (fn [] ,desc ,...)))
+      ,desc (fn [] ,desc ,(unpack body)))))
 
 (fn before-each [...]
   "busted's `describe` function"
+  (let [body [...]]
+    (table.insert body `nil)
     `((. (require :busted) :before_each)
-    (fn [] ,...)))
+      (fn [] ,(unpack body)))))
 
 
 (fn is-matching [item pattern ?msg]