find definition is more advanced but more brittle

Makefile

@@ -1,13 +1,21 @@
 LUA_LIB=/usr/lib/liblua.so.5.4
 LUA_INCLUDE_DIR=/usr/include/lua5.4
-SOURCES=$(wildcard src/*.fnl)
-SOURCES+=$(wildcard src/fennel-ls/*.fnl)
+FENNEL=./fennel
+EXE=fennel-ls
 
-.PHONY: test
+SRC=$(wildcard src/*.fnl)
+SRC+=$(wildcard src/fennel-ls/*.fnl)
+
+.PHONY: clean test
+
+all: $(EXE)
+
+$(EXE): $(SRC)
+	LUA_PATH="./src/?.lua;./src/?/init.lua" FENNEL_PATH="./src/?.fnl;./src/?/init.fnl" $(FENNEL) --compile-binary src/fennel-ls.fnl fennel-ls $(LUA_LIB) $(LUA_INCLUDE_DIR)
 
-fennel-ls: $(SOURCES)
-	LUA_PATH="./src/?.lua;./src/?/init.lua" FENNEL_PATH="./src/?.fnl;./src/?/init.fnl" ./fennel --compile-binary src/fennel-ls.fnl fennel-ls $(LUA_LIB) $(LUA_INCLUDE_DIR)
 clean:
 	rm -f fennel-ls
+
 test:
-	FENNEL_PATH="./src/?.fnl;./src/?/init.fnl" ./fennel --correlate test/init.fnl --verbose
+	# requires busted to be installed
+	FENNEL_PATH="./src/?.fnl;./src/?/init.fnl" $(FENNEL) --correlate test/init.fnl --verbose

README.md

@@ -4,28 +4,30 @@ THIS PROJECT IS NOT IN A USABLE STATE YET. CHECK BACK LATER.
 
 A language server for fennel-ls.
 
-(Planned) Features:
-[ ] for planned features
-[X] for implemented features
+Features / To Do List / Things I would enjoy patches for:
 
-* [X] Able to connect to a client
-* [ ] Support for UTF-8 characters that aren't just plain ASCII. (especially `λ`)
-* [ ] Settings to configure lua / fennel path, allowed globals, etc
-* [ ] Builds for anything other than arch linux
-* [X] Go-to-definition for (require) statements
-* [X] basic go-to-definition for in-file definitions
-* [ ] Go-to-definition for in-file definitions in all cases
-* [ ] Go-to-definition for definitions in other files
-* [ ] Go-to-definition into lua code
-* [ ] Reports compiler errors
-* [ ] Reports linting issues
-* [ ] basic completion suggestions
-* [ ] Hover over a symbol for documentation
-* [ ] Signature help
-* [ ] Go-to-references on definitions of things
-* [ ] integration with fnlfmt
-* [ ] Maybe some sort of type checking???
-    * [ ] completion suggestions for fields / methods
+- [X] Able to connect to a client
+- [ ] Support for UTF-8 characters that aren't just plain ASCII. (especially `λ`)
+- [ ] Settings to configure lua / fennel path, allowed globals, etc
+- [ ] Builds for anything other than arch linux
+- [ ] Go-to-definition
+    - [ ] directly on require statements
+    - [X] for definitions in the same file
+    - [ ] for definitions in other files
+    - [ ] follows multisyms through table constructor
+    - [ ] follows multisyms through mutations (difficult)
+    - [ ] for methods/metamethods (difficult, in the general case may require type annotations or some insane global type inference logic)
+    - [ ] into lua files (maybe cheeseable with antifennel if it has --correlate?)
+
+- [ ] Reports compiler errors
+    - [ ] including in macro files
+- [ ] Reports linting issues
+- [ ] Completion Suggestions
+- [ ] Hover over a symbol for documentation
+- [ ] Signature help
+- [ ] Go-to-references on definition sites
+- [ ] integration with fnlfmt
+- [ ] Maybe some sort of type checking??
 
 
 ## Setup:

file.fnl

@@ -0,0 +1,2 @@
+(match [0 10]
+  [1 a] a)

src/fennel-ls/parser.fnl

@@ -1,31 +1,42 @@
 (local fennel (require :fennel))
 (local util (require :fennel-ls.util))
 
-(fn get-ast-info [ast info]
+(λ get-ast-info [?ast info]
   "find a given key of info from an AST object"
-  (or (. (getmetatable ast) info)
-      (. ast info)))
+  (or (?. (getmetatable ?ast) info)
+      (. ?ast info)))
 
-(fn contains? [ast byte]
+(λ contains? [?ast byte]
   "check if a byte is in range of the AST object"
-  (and (= (type ast) :table)
-       (get-ast-info ast :bytestart)
-       (get-ast-info ast :byteend)
-       (<= (get-ast-info ast :bytestart)
+  (and (= (type ?ast) :table)
+       (get-ast-info ?ast :bytestart)
+       (get-ast-info ?ast :byteend)
+       (<= (get-ast-info ?ast :bytestart)
            byte
-           (get-ast-info ast :byteend))))
+           (+ 1 (get-ast-info ?ast :byteend)))))
 
-(fn past? [ast byte]
+(λ does-not-contain? [?ast byte]
+  "check if a byte is in range of the AST object"
+  (and (= (type ?ast) :table)
+       (get-ast-info ?ast :bytestart)
+       (get-ast-info ?ast :byteend)
+       (not
+         (<= (get-ast-info ?ast :bytestart)
+             byte
+             (+ 1 (get-ast-info ?ast :byteend))))))
+
+
+(λ past? [?ast byte]
   "check if a byte is past the range of the AST object"
-  (and (= (type ast) :table)
-       (get-ast-info ast :bytestart)
-       (< byte (get-ast-info ast :bytestart))
+  (and (= (type ?ast) :table)
+       (get-ast-info ?ast :bytestart)
+       (< byte (get-ast-info ?ast :bytestart))
        false))
 
-(fn range [text ast]
+(λ range [text ?ast]
   "create a LSP range representing the span of an AST object"
-  (if (= (type ast) :table)
-    (match (values (get-ast-info ast :bytestart) (get-ast-info ast :byteend))
+  (if (= (type ?ast) :table)
+    (match (values (get-ast-info ?ast :bytestart) (get-ast-info ?ast :byteend))
       (i j)
       (let [(start-line start-col) (util.byte->pos text i)
             (end-line   end-col)   (util.byte->pos text (+ j 1))]
@@ -33,5 +44,6 @@
          :end   {:line end-line   :character end-col}}))))
 
 {: contains?
+ : does-not-contain?
  : past?
  : range}

src/fennel-ls/plugin.fnl

@@ -15,19 +15,28 @@
 (λ table? [t]
   (= :table (type t)))
 
+(λ string? [t]
+  (= :string (type t)))
+
 (λ multisym? [t]
   (and (fennel.sym? t)
     (let [t (tostring t)]
        (or (t:find "%.")
            (t:find ":")))))
 
+(λ iter [t]
+  (if (or (fennel.list? t)
+          (fennel.sequence? t))
+    (ipairs t)
+    (pairs t)))
+
 
 (λ analyze [file]
   (assert file.text (fennel.view file))
   (assert file.uri)
   (set file.references [])
 
-  (local scope-notes
+  (local definitions
     (doto {}
       (setmetatable
         {:__index
@@ -36,27 +45,29 @@
              (tset self key val)
              val))})))
 
-  (λ find-reference [name ?scope]
+  (λ find-variable [name ?scope]
     (when ?scope
-      (or (. scope-notes ?scope (tostring name))
-          (find-reference name ?scope.parent))))
+      (or (. definitions ?scope name)
+          (find-variable name ?scope.parent))))
 
   (λ reference [ast scope]
     "called whenever a variable is referenced"
     (assert (fennel.sym? ast))
+    ;; find reference
     (let [name (string.match (tostring ast) "[^%.:]+")
-          target (find-reference name scope)]
-      (table.insert file.references {:from ast :to target})))
+          target (find-variable (tostring name) scope)]
+      (tset file.references ast target)))
 
   (λ define [?definition binding scope]
     "called whenever a local variable or destructure statement is introduced"
     ;; right now I'm not keeping track of *how* the symbol was destructured: just finding all the symbols for now.
     (λ recurse [binding]
       (if (fennel.sym? binding)
-        ;; this seems to defeat match's symbols in specifically the one case I've tested
-        (if (not (?. scope :parent :gensyms (tostring binding)))
-          (tset (. scope-notes scope) (tostring binding) binding))
-        (each [k v ((if (fennel.list? binding) ipairs pairs) binding)]
+        (tset (. definitions scope)
+              (tostring binding)
+              {: binding :definition ?definition})
+        (table? binding)
+        (each [k v (iter binding)]
          (recurse v))))
     (recurse binding))
 
@@ -66,7 +77,10 @@
         (and (fennel.sym? name)
              (not (multisym? name)) ;; not dealing with multisym for now
              (fennel.sequence? args)))
-      (tset (. scope-notes scope.parent) (tostring name) ast)))
+      (tset (. definitions scope.parent)
+            (tostring name)
+            {:binding name
+             :definition ast})))
 
   (λ define-function-args [ast scope]
     (local args
@@ -79,15 +93,13 @@
   (λ define-function [ast scope]
     "Introduces the various symbols exported by a function.
 This cannot be done through the :fn feature of the compiler plugin system, because it needs to be
-called before the body of the function happens"
+called *before* the body of the function is processed."
     (define-function-name ast scope)
     (define-function-args ast scope))
 
   (λ call [ast scope]
-    "called for every function call. Most calls aren't interesting, but (require) and (local) are"
+    "called for every function call. Most calls aren't interesting, but fn is"
     (match ast
-      (where [-require- mod] (= :string (type mod)))
-      (insert file.references {:from ast :to-other-module [mod]})
       [-fn-]
       (define-function ast scope)
       [-λ-]
@@ -102,8 +114,13 @@ called before the body of the function happens"
      : call
      :destructure define})
 
-  (pcall fennel.compileString file.text
-    {:filename file.uri
-     :plugins [plugin]}))
+  (set file.ast (icollect [ok ast (fennel.parser file.text)]
+                 ast))
+  (local scope (fennel.scope))
+  (each [_i form (ipairs file.ast)]
+    (fennel.compile form
+      {:filename file.uri
+       : scope
+       :plugins [plugin]})))
 
 {: analyze}

src/fennel-ls/the-actual-code.fnl

@@ -5,6 +5,9 @@ Every time the client sends a message, it gets handled by a function in the corr
 (ie, a textDocument/didChange notification will call notifications.textDocument/didChange
  and a textDocument/defintion request will call requests.textDocument/didChange)"
 (local fennel (require :fennel))
+(local sym? fennel.sym?)
+(local list? fennel.list?)
+(local fennelutils (require :fennel.utils))
 
 (local parser  (require :fennel-ls.parser))
 (local util    (require :fennel-ls.util))
@@ -61,26 +64,94 @@ Every time the client sends a message, it gets handled by a function in the corr
   {:capabilities capabilities
    :serverInfo {:name "fennel-ls" :version "0.0.0"}})
 
-(fn string? [j]
+(λ string? [j]
   (= (type j) :string))
 
-(local require* (fennel.sym :require))
-(local local* (fennel.sym :local))
+(λ get-assignment-of-symbol [file symbol]
+  ;; TODO inline
+  (. file.references symbol))
+
+;; These three functions are mutually recursive
+(var (search-item
+      search-assignment
+      search-symbol)
+  nil)
+
+(set search-item
+  (λ search-item [self file item stack]
+    (if ;; table
+        (fennelutils.table? item)
+        (if (. item (. stack (length stack)))
+          (search-item self file (. item (table.remove stack)) stack)
+          nil)
+        ;; symbol
+        (sym? item)
+        (search-symbol self file item stack)
+        ;; TODO
+        ;; functioncall (into body)
+        ;; require functioncall (into module)
+
+        ;; else
+        true (error (.. "I don't know what to do with " (fennel.view item))))))
+
+(set search-assignment
+  (λ search-assignment [self file binding ?definition stack]
+    (if (= 0 (length stack))
+      binding
+      ;; TODO sift down the binding
+      (search-item self file ?definition stack))))
+
+(set search-symbol
+  (λ search-symbol [self file symbol stack]
+    (let [split (util.multi-sym-split symbol)]
+      (for [i (length split) 2 -1]
+        (table.insert stack (. split i))))
+    (match (get-assignment-of-symbol file symbol)
+      to (search-assignment self file to.binding to.definition stack)
+      nil nil)))
+
+(λ iter [t]
+  (if (or (fennel.list? t)
+          (fennel.sequence? t))
+    (ipairs t)
+    (pairs t)))
+
+(λ find-symbol* [ast byte]
+  (if (not= :table (type ast))
+      nil
+      (parser.does-not-contain? ast byte)
+      nil
+      (sym? ast)
+      ast
+      (or (fennel.list? ast)
+          (fennel.sequence? ast))
+      ;; TODO binary search
+      (accumulate [result nil
+                   _ v (ipairs ast) &until (or result (parser.past? v byte))]
+        (find-symbol* v byte))
+      :else (accumulate [result nil
+                         k v (pairs ast) &until result]
+              (or
+                (find-symbol* k byte)
+                (find-symbol* v byte)))))
+
+(λ find-symbol [ast byte]
+  ;; TODO binary search
+  (accumulate [result nil
+               _ v (ipairs ast) &until (or result (parser.past? v byte))]
+    (find-symbol* v byte)))
+
 (λ requests.textDocument/definition [self send {: position :textDocument {: uri}}]
   (local file (state.get-by-uri self uri))
   (local byte (util.pos->byte file.text position.line position.character))
-  (accumulate [result nil
-               _ reference (ipairs file.references) &until (or result (parser.past? reference.from byte))]
-      (if (parser.contains? reference.from byte)
-        (match reference
-          {: from : to}
-          {:range (parser.range file.text to)
-           :uri file.uri}
-
-          {: from : to-other-module}
-          {:range {:start {:line 0 :character 0}
-                   :end   {:line 0 :character 0}}
-           :uri (mod.lookup self (. to-other-module 1))}))))
+  (local stack [])
+  (match (find-symbol file.ast byte)
+    symbol
+    (match (search-symbol self file symbol stack)
+      definition
+      {:range (parser.range file.text definition)
+       :uri uri})
+    nil nil))
 
 (λ notifications.textDocument/didChange [self send {: contentChanges :textDocument {: uri}}]
   (local file (state.get-by-uri self uri))

src/fennel-ls/util.fnl

@@ -69,8 +69,21 @@ These functions are all pure functions, which makes me happy."
       {: text}
       text)))
 
+(fn multi-sym-split [sym ?offset]
+  (local sym (tostring sym))
+  (local offset (or ?offset (length sym)))
+  (local next-separator (or (sym:find ".[%.:]" offset)
+                            (length sym)))
+  (local sym (sym:sub 1 next-separator))
+  (icollect [word (: (.. sym ".") :gmatch "(.-)[%.:]")]
+    word))
+
+(fn reversed [tab])
+
+
 {: uri->path
  : path->uri
  : pos->byte
  : byte->pos
- : apply-changes}
+ : apply-changes
+ : multi-sym-split}

test.fnl

@@ -0,0 +1,10 @@
+(var (a b c) nil)
+
+
+(set a (fn a []
+        (b)))
+
+(set b (fn b []
+        (print "yay")))
+
+(a)

test/goto-definition-test.fnl

@@ -1,5 +1,5 @@
-(import-macros {: assert-matches : describe : it : before-each} :test.macros)
-(local assert (require :luassert))
+(import-macros {: is-matching : describe : it : before-each} :test.macros)
+(local is (require :luassert))
 
 (local fennel (require :fennel))
 (local {: ROOT-URI
@@ -17,7 +17,7 @@
                        {:position {:character char :line line}
                         :textDocument {:uri (.. ROOT-URI "/" request-file)}}))
           uri (.. ROOT-URI "/" response-file)]
-      (assert-matches
+      (is-matching
         message
         [{:jsonrpc "2.0" :id 2
           :result {: uri
@@ -25,15 +25,8 @@
                            :end   {:line end-line   :character end-col}}}}]
         (.. "expected position: " start-line " " start-col " " end-line " " end-col))))
 
-  (it "handles (local _ (require XXX)"
-    (check "example.fnl" 0 11 "foo.fnl" 0 0 0 0))
-
-  (it "handles (require XXX))"
-    (check "example.fnl" 1 5 "bar.fnl" 0 0 0 0))
-
   (it "can go to a fn"
-    ;; TODO maybe it's better to just go to the name of the function, not the whole list
-    (check "example.fnl" 9 3 "example.fnl" 4 0 7 20))
+    (check "example.fnl" 9 3 "example.fnl" 4 4 4 7))
 
   (it "can go to a local"
     (check "example.fnl" 7 17 "example.fnl" 6 9 6 10))
@@ -48,11 +41,20 @@
     (check "example.fnl" 19 12 "example.fnl" 17 8 17 9))
 
   (it "can sort out the unification rule with match (variable introduced)"
-    (check "example.fnl" 20 12 "example.fnl" 20 9 20 10))
+    (check "example.fnl" 20 13 "example.fnl" 20 9 20 10))
 
   (it "can go to a destructured local"
-    (check "example.fnl" 21 9 "example.fnl" 16 13 16 16)))
-  ;; (it "can go to a function inside a table")
+    (check "example.fnl" 21 9 "example.fnl" 16 13 16 16))
+
+  (it "can go to a function inside a table"
+    (check "example.fnl" 28 6 "example.fnl" 4 4 4 7)))
+
+  ;; (it "handles (local _ (require XXX)"
+  ;;   (check "example.fnl" 0 11 "foo.fnl" 0 0 0 0))
+
+  ;; (it "handles (require XXX))"
+  ;;   (check "example.fnl" 1 5 "bar.fnl" 0 0 0 0))
+
   ;; (it "can go to a field inside of a table")
   ;; (it "can go to a destructured function argument")
   ;; (it "can go to a reference that occurs in a macro")

test/init.fnl

@@ -4,3 +4,4 @@
 (require :test.string-processing-test)
 (require :test.lsp-test)
 (require :test.goto-definition-test)
+(require :test.misc-test)

test/json-rpc-test.fnl

@@ -1,5 +1,5 @@
-(import-macros {: assert-matches : describe : it} :test.macros)
-(local assert (require :luassert))
+(import-macros {: is-matching : describe : it} :test.macros)
+(local is (require :luassert))
 
 (local fennel (require :fennel))
 (local stringio (require :test.pl.stringio))
@@ -10,30 +10,30 @@
     (it "parses incoming messages"
       (let [out (stringio.open
                   "Content-Length: 29\r\n\r\n{\"my json content\":\"is cool\"}")]
-        (assert.same
+        (is.same
           {"my json content" "is cool"}
           (json-rpc.read out))))
 
     (it "can read multiple incoming messages"
       (let [out (stringio.open
                   "Content-Length: 29\r\n\r\n{\"my json content\":\"is cool\"}Content-Length: 29\r\n\r\n{\"my json content\":\"is neat\"}")]
-        (assert.same
+        (is.same
           {"my json content" "is cool"}
           (json-rpc.read out))
-        (assert.same
+        (is.same
           {"my json content" "is neat"}
           (json-rpc.read out))
-        (assert.same
+        (is.same
           nil
           (json-rpc.read out))))
 
     (it "can report compiler errors"
       (let [out (stringio.open "Content-Length: 9\r\n\r\n{{{{{}}}}")]
-        (assert (= (type (json-rpc.read out)) :string)))))
+        (is (= (type (json-rpc.read out)) :string)))))
 
   (describe "write"
     (it "serializes outgoing messages"
       (let [in (stringio.create)]
         (json-rpc.write in {"my json content" "is cool"})
-        (assert.same "Content-Length: 29\r\n\r\n{\"my json content\":\"is cool\"}"
+        (is.same "Content-Length: 29\r\n\r\n{\"my json content\":\"is cool\"}"
                      (in:value))))))

test/lsp-test.fnl

@@ -1,5 +1,5 @@
-(import-macros {: assert-matches : describe : it} :test.macros)
-(local assert (require :luassert))
+(import-macros {: is-matching : describe : it} :test.macros)
+(local is (require :luassert))
 
 (local {: ROOT-PATH : ROOT-URI} (require :test.util))
 (local dispatch (require :fennel-ls.dispatch))
@@ -21,7 +21,7 @@
 
 (describe "language server"
   (it "responds to initialize"
-    (assert-matches
+    (is-matching
       (dispatch.handle* [] server-initialize-message)
       [{:jsonrpc "2.0" :id 1
         :result {:capabilities {}

test/macros.fnl

@@ -16,7 +16,7 @@
     (fn [] ,...)))
 
 
-(fn assert-matches [item pattern ?msg]
+(fn is-matching [item pattern ?msg]
   "check if item matches a pattern according to fennel's `match` builtin"
   `(match ,item
     ,pattern nil
@@ -31,5 +31,5 @@
 
 {: it
  : describe
- : assert-matches
+ : is-matching
  : before-each}

test/misc-test.fnl

@@ -0,0 +1,22 @@
+(import-macros {: is-matching : describe : it : before-each} :test.macros)
+(local is (require :luassert))
+
+(local fennel (require :fennel))
+(local {: multi-sym-split} (require :fennel-ls.util))
+
+(describe "multi-sym-split"
+  (it "should be 1 on regular syms"
+    (is.same ["foo"] (multi-sym-split "foo" 2)))
+
+  (it "should be 1 before the :"
+    (is.same ["foo"] (multi-sym-split "foo:bar" 3)))
+
+  (it "should be 2 at the :"
+    (is.same ["foo" "bar"] (multi-sym-split "foo:bar" 4)))
+
+  (it "should be 2 after the :"
+    (is.same ["is" "equal"] (multi-sym-split "is.equal" 5)))
+
+  (it "should be big"
+    (is.same ["a" "b" "c" "d" "e" "f"] (multi-sym-split "a.b.c.d.e.f"))
+    (is.same ["obj" "bar"] (multi-sym-split (fennel.sym "obj.bar")))))

test/string-processing-test.fnl

@@ -1,10 +1,10 @@
-(import-macros {: assert-matches : describe : it} :test.macros)
-(local assert (require :luassert))
+(import-macros {: is-matching : describe : it} :test.macros)
+(local is (require :luassert))
 
 (local fennel (require :fennel))
 (local util (require :fennel-ls.util))
 
-(describe "document"
+(describe "util"
 
     ;; fixme:
     ;; test for errors on out of bounds
@@ -17,7 +17,7 @@
     ;;   (document.replace my-document 0 0 0 0 "どれみふぁそらてぃど")
     ;;   (document.replace my-document 0 1 0 3 "😀")
     ;;   (document.replace my-document 0 11 0 11 "end")
-    ;;   (assert-matches my-document {:text "ど😀ふぁそらてぃどend"})))
+    ;;   (is-matching my-document {:text "ど😀ふぁそらてぃどend"})))
 
   (describe "apply-changes"
 
@@ -26,7 +26,7 @@
        :end   {:line end-line   :character end-col}})
 
     (it "updates the start of a line"
-      (assert.equal
+      (is.equal
         (util.apply-changes
           "replace beginning"
           [{:range (range 0 0 0 7)
@@ -34,7 +34,7 @@
         "the beginning"))
 
     (it "updates the end of a line"
-      (assert.equal
+      (is.equal
         (util.apply-changes
           "first line\nsecond line\nreplace end"
           [{:range (range 2 7 2 11)
@@ -42,7 +42,7 @@
         "first line\nsecond line\nreplacement"))
 
     (it "replaces a line"
-      (assert.equal
+      (is.equal
         (util.apply-changes
           "replace all"
           [{:range (range 0 0 0 11)
@@ -50,7 +50,7 @@
         "new string"))
 
     (it "can handle substituting things"
-      (assert.equal
+      (is.equal
         (util.apply-changes
           "replace beginning"
           [{:range {:start {:line 0 :character 0}
@@ -59,7 +59,7 @@
         "the beginning"))
 
     (it "can handle replacing everything"
-      (assert.equal
+      (is.equal
         (util.apply-changes
           "this is the\nold file"
           [{:text "And this is the\nnew file"}])

test/test-project/example.fnl

@@ -21,4 +21,10 @@
       [0 b] b))
   (print foo))
 
-{: bar}
+(fn b []
+  (print "function"))
+
+(local obj {: bar :a b})
+
+(obj.bar 2 3)
+(obj:a)