Refactoring / documentation comments / small fixes

src/fennel-ls/compiler.fnl

@@ -1,3 +1,8 @@
+"Compiler
+This file is responsible for the low level tasks of analysis. Its main job
+is to recieve a file object and run all of the basic analysis that will be used
+later by fennel-ls.language to answer requests from the client."
+
 (local {: sym? : list? : sequence? : sym : view &as fennel} (require :fennel))
 (local message (require :fennel-ls.message))
 
@@ -32,6 +37,7 @@
 
 (λ compile [file]
   "Compile the file, and record all the useful information from the compiler into the file object"
+  ;; The useful information being recorded:
   (let [definitions-by-scope (doto {} (setmetatable has-tables-mt))
         definitions   {}
         diagnostics   {}
@@ -115,8 +121,9 @@
         [-require- modname]
         (tset require-calls ast true)))
 
-    (λ on-compiler-error [_ msg ast call-me-to-reset-the-compiler]
+    (λ on-compile-error [_ msg ast call-me-to-reset-the-compiler]
-      (let [range (message.ast->range ast file)]
+      (let [range (or (message.ast->range ast file)
+                      (message.pos->range 0 0 0 0))]
         (table.insert diagnostics
           {:range range
            :message msg
@@ -126,6 +133,17 @@
       (call-me-to-reset-the-compiler)
       (error "__NOT_AN_ERROR"))
 
+    (λ on-parse-error [msg file line byte]
+      ;; assume byte and char count is the same, ie no UTF-8
+      (let [range (message.pos->range line byte line byte)]
+        (table.insert diagnostics
+          {:range range
+           :message msg
+           :severity message.severity.ERROR
+           :code 201
+           :codeDescription "compiler error"}))
+      (error "__NOT_AN_ERROR"))
+
     ;; TODO clean up this code. It's awful now that there is error handling
     (let
       [plugin
@@ -134,31 +152,23 @@
         :symbol-to-expression reference
         :call call
         :destructure define
-        :assert-compile on-compiler-error}]
+        :assert-compile on-compile-error
-
+        :parse-error on-parse-error}
-      ;; ATTEMPT TO PARSE AST
+       scope (fennel.scope)
-      (match (pcall
+       opts {:filename file.uri
-               #(icollect [ok ast (fennel.parser file.text file.uri {:plugins [plugin]})]
-                  ast))
-        ;; ON SUCCESS
-        (true ast)
-        (let [scope (fennel.scope)]
-          (each [_i form (ipairs ast)]
-            ;; COMPILE
-            (match (pcall fennel.compile form {:filename file.uri
              :plugins [plugin]
              :allowedGlobals (icollect [k v (pairs _G)] k)
              :requireAsInclude false
-                                               : scope})
+             : scope}
+       parser (partial pcall (fennel.parser file.text file.uri opts))
+       ast (icollect [ok ok2 ast parser &until (not (and ok ok2))] ast)
+       _compile-output (icollect [_i form (ipairs ast)]
+                         (match (pcall fennel.compile form opts)
                            (where (nil err) (not= err "__NOT_AN_ERROR"))
-              (error err)))
-          (set file.ast ast))
-        ;; ON FAILURE
-        (false err)
-        ;; RECORD THE FAILURE
                            (table.insert diagnostics
                              {:range (message.pos->range 0 0 0 0)
-           :message err}))
+                              :message err})))]
+      (set file.ast ast)
 
 
       ;; write things back to the file object

src/fennel-ls/formatter.fnl

@@ -1,3 +1,8 @@
+"Formatter
+This module is for formatting code that needs to be shown to the client
+in tooltips and other notification messages. It is NOT for formatting
+user code."
+
 (local {: sym
         : sym?
         : view
@@ -20,6 +25,7 @@
       (if docstring (.. "\n" docstring) "")))
 
 (λ hover-format [result]
+  "Format code that will appear when the user hovers over a symbol"
   (code-block
       (match result.?definition
         ;; name + docstring

src/fennel-ls/language.fnl

@@ -1,3 +1,7 @@
+"Language
+The high level analysis system that does deep searches following
+the data provided by compiler.fnl."
+
 (local {: sym? : list? : sequence? : sym : view} (require :fennel))
 (local utils (require :fennel-ls.utils))
 (local state (require :fennel-ls.state))

src/fennel-ls/state.fnl

@@ -1,3 +1,10 @@
+"State
+This module keeps track of the state of the language server.
+There are helpers to get file objects, and in the future, there
+will be functions for managing user options. There is no global
+state in this project: all state will be stored in the \"self\"
+object."
+
 (local utils (require :fennel-ls.utils))
 (local searcher (require :fennel-ls.searcher))
 (local {: compile} (require :fennel-ls.compiler))

src/fennel-ls/utils.fnl

@@ -75,14 +75,18 @@ These functions are all pure functions, which makes me happy."
   (or (?. (getmetatable ?ast) info)
       (. ?ast info)))
 
-(fn multi-sym-split [sym ?offset]
+(fn multi-sym-split [symbol ?offset]
-  (local sym (tostring sym))
+  (local symbol (tostring symbol))
-  (local offset (or ?offset (length sym)))
+  (if (or (= symbol ".")
-  (local next-separator (or (sym:find ".[%.:]" offset)
+          (= symbol "..")
-                            (length sym)))
+          (= symbol "..."))
-  (local sym (sym:sub 1 next-separator))
+    [symbol]
-  (icollect [word (: (.. sym ".") :gmatch "(.-)[%.:]")]
+    (let [offset (or ?offset (length symbol))
-    word))
+          next-separator (or (symbol:find ".[.:]" offset)
+                             (length symbol))
+          symbol (symbol:sub 1 next-separator)]
+      (icollect [word (: (.. symbol ".") :gmatch "(.-)[.:]")]
+        word))))
 
 (λ type= [val typ]
   (= (type val) typ))

test/completion-test.fnl

@@ -0,0 +1,36 @@
+(import-macros {: is-matching : describe : it : before-each} :test)
+(local is (require :luassert))
+
+(local {: view} (require :fennel))
+(local {: ROOT-URI
+        : setup-server} (require :test.util))
+
+(local dispatch (require :fennel-ls.dispatch))
+(local message (require :fennel-ls.message))
+
+(local FILENAME (.. ROOT-URI "imaginary-file.fnl"))
+
+(fn open-file [state text]
+  (dispatch.handle* state
+    (message.create-notification "textDocument/didOpen"
+      {:textDocument
+       {:uri FILENAME
+        :languageId "fennel"
+        :version 1
+        : text}})))
+
+(describe "completions")
+  ;; (it "suggests globals"
+  ;;   (local state (doto [] setup-server))
+  ;;   ;; empty file
+  ;;   (open-file state "")
+  ;;   (let [response (dispatch.handle* state
+  ;;                    (message.create-request 2 "textDocument/completion"
+  ;;                      {:position {:line 0 :character 0}
+  ;;                       :textDocument {:uri FILENAME}}))]
+  ;;     (is-matching response nil "oops"))))
+
+  ;; (it "suggests locals in scope")
+  ;; (it "does not suggest locals out of scope")
+  ;; (it "suggests fields of tables")
+  ;; (it "knows what fields are meant to be inside of globals")

test/diagnostic-test.fnl

@@ -8,35 +8,69 @@
 (local dispatch (require :fennel-ls.dispatch))
 (local message (require :fennel-ls.message))
 
+(macro find [t body ?sentinel]
+  (assert-compile (not ?sentinel) "you can only have one thing here, put a `(do)`")
+  (assert-compile (sequence? t) "[] square brackets please")
+  (local result (gensym :result))
+  (local nil* (sym :nil))
+  (table.insert t 1 result)
+  (table.insert t 2 nil*)
+  (table.insert t `&until)
+  (table.insert t result)
+  `(accumulate ,t ,body))
+
+(fn open-file [state text]
+  (dispatch.handle* state
+    (message.create-notification "textDocument/didOpen"
+      {:textDocument
+       {:uri (.. ROOT-URI "imaginary-file.fnl")
+        :languageId "fennel"
+        :version 1
+        : text}})))
+
 (describe "diagnostic messages"
   (it "handles compile errors"
     (local state (doto [] setup-server))
-    (let
+    (let [responses (open-file state "(do do)")
-      [responses
+          diagnostic
-       (dispatch.handle* state
+          (match responses
-         (message.create-notification "textDocument/didOpen"
+            [{:params {: diagnostics}}]
-           {:textDocument
+            (find [i v (ipairs diagnostics)]
-            {:uri (.. ROOT-URI "imaginary-file.fnl")
+               (match v
-             :languageId "fennel"
+                 {:message "tried to reference a special form at runtime"
-             :version 1
+                  :range {:start {:character 4 :line 0}
-             :text "(do do)"}}))]
+                          :end   {:character 6 :line 0}}}
-      (is-matching
+                 v)))]
-        responses
+      (is diagnostic "expected a diagnostic")))
-        [{:params {:diagnostics [diagnostic]}}]
-        "")))
 
   (it "handles parse errors"
     (local state (doto [] setup-server))
-    (let
+    (let [responses (open-file state "(do (print :hello(]")
-      [responses
+          diagnostic
-       (dispatch.handle* state
+          (match responses
-         (message.create-notification "textDocument/didOpen"
+            [{:params {: diagnostics}}]
-           {:textDocument
+            (find [i v (ipairs diagnostics)]
-            {:uri (.. ROOT-URI "imaginary-file.fnl")
+             (match v
-             :languageId "fennel"
+               {:message "expected whitespace before opening delimiter ("
-             :version 1
+                :range {:start {:character 17 :line 1}
-             :text "(do (print :hello(]"}}))]
+                        :end   {:character 17 :line 1}}}
-      (is-matching
+               v)))]
-        responses
+      (is diagnostic "expected a diagnostic")))
-        [{:params {:diagnostics [diagnostic]}}]
+
-        ""))))
+  (it "handles (match)"
+    (local state (doto [] setup-server))
+    (let [responses (open-file state "(match)")]
+      (is-matching responses
+        [{:params
+          {:diagnostics
+           [{:range {:start {:character a :line b}
+                     :end   {:character c :line d}}}]}}]
+        "diagnostics should always have a range"))))
+
+;; TODO lints:
+;; unnecessary (do) in body position
+;; Unused variables / fields (maybe difficult)
+;; discarding results to various calls
+;; unnecessary `do`/`values` with only one inner form
+;; mark when unification is happening on a `match` pattern (may be difficult)
+;; think of more lints

test/goto-definition-test.fnl

@@ -70,12 +70,18 @@
 
   ;; TODO
   ;; (it "can go to a function in another file imported via destructuring assignment") ;; WORKS, just needs a test case
-  ;; (it "doesn't have ghost definitions from the same byte ranges as the macro files it's using") ;; Unconfirmed, I saw some weird behavior a while ago
   ;; (it "can go through more than one extra file")
   ;; (it "will give up instead of freezing on recursive requires")
   ;; (it "finds the definition of in-file macros")
   ;; (it "can follow import-macros (destructuring)")
   ;; (it "can follow import-macros (namespaced)")
   ;; (it "can go to the definition even in a lua file")
-  ;; (it "can go to a function's arguments when they're available")
+  ;; (it "finds (fn a.b [] ...) declarations")
+  ;; (it "finds (set a.b) definitions")
+  ;; (it "finds (tset a :b) definitions")
+  ;; (it "finds (setmetatable a {__index {:b def}) definitions")
+  ;; (it "finds definitions from inside a function (fn foo [] (local x 10) {: x}) (let [result (foo)] (print result.x)) finds result.x")
+  ;; (it "finds basic setmetatable definitions with an __index function")
+  ;; (it "can return to callsite and go through a function's arguments when they're available")
+  ;; (it "can go to a function's reference OR read type inference comments when callsite isn't available (PICK ONE)")
   ;; (it "can work with a custom fennelpath") ;; Wait until an options system is done

test/init.fnl

@@ -1,9 +1,10 @@
 ((require :busted.runner))
 
-(require :test.json-rpc-test)
+(require :test.completion-test)
-(require :test.string-processing-test)
+(require :test.diagnostic-test)
-(require :test.lsp-test)
 (require :test.goto-definition-test)
 (require :test.hover-test)
+(require :test.json-rpc-test)
+(require :test.lsp-test)
 (require :test.misc-test)
-(require :test.diagnostic-test)
+(require :test.string-processing-test)