diff --git a/make-docs.fnl b/make-docs.fnl index 5696195..97c07d8 100644 --- a/make-docs.fnl +++ b/make-docs.fnl @@ -1,39 +1,86 @@ -"work in progress script to generate /src/fennel-ls/docs/lua54.fnl automatically" -(local {: view} (require :fennel)) -(local version "5.4") -(local filename (.. "lua" version ".html")) -(var file (io.open filename :r)) -(when (not file) - ;; I think the repos also have the html file (or the ability to generate it, depending on lua version) - ;; but for simplicity I'm just grabbing the html straight from the website - (os.execute (.. "curl https://www.lua.org/manual/" version "/manual.html > lua" version ".html")) - (set file (io.open filename :r))) +"Script to generate /src/fennel-ls/docs/lua54.fnl and friends automatically" -(local html (file:read :a)) +(fn open-file-cached [filename url] + (var file (io.open filename :r)) + (when (not file) + ;; I think the repos also have the html file (or the ability to generate it, depending on lua version) + ;; but for simplicity I'm just grabbing the html straight from the website + (print (.. "Downloading " url)) + (os.execute (.. "curl " url " > " filename)) + (set file (io.open filename :r))) + file) -;; The section of the html about the standard library -(local begin-index (assert (html:find "

.%.1 – Basic Functions.-\n"))) -(local end-index (assert (html:find "

. – .%.1 – Basic Functions.-\n") "no basic functions?") + end-index (assert (html:find "

. –

.-\n" (+ prev 1)) + section (stdlib:sub prev (if header (- header 1)))] + (let [index (section:find "

")] + (if index + (do + (table.insert fields (section:sub 1 (- index 1))) + (table.insert modules (section:sub index))) + (table.insert fields section))) + (when header (loop header)))) + (loop (stdlib:find "

.-\n")) + (values modules fields))) +(fn html-to-markdown [str] + (let [str + (-> str + ;; delete

tags + (: :gsub "" "") + ;; tags for the rest + (: :gsub "\"(.-)\"" "`\"%1\"`") + (: :gsub "\'(.-)\'" "`\"%1\"`") + (: :gsub "(.-)" "`%1`") + (: :gsub "x" "ˣ") + (: :gsub "e" "ᵉ") + (: :gsub "y" "ʸ") + (: :gsub "51" "⁵¹") + (: :gsub "32" "³²") + ; ᵃᵇᶜᵈᵉᶠᵍʰⁱʲᵏˡᵐⁿᵒᵖ𐞥ʳˢᵗᵘᵛʷˣʸᶻ + ;; bold to ** + (: :gsub "([^<]+)" "*%1*") + (: :gsub "([^<]+)" "**%1**") + ;; defeat all the links + (: :gsub "([^<]+)" "%1") + (: :gsub "([^<]+)" "%1") + (: :gsub "([^<]+)" "%1") + (: :gsub "See [^<]+[^%.]+%." "") + (: :gsub "[ \n]%(see [^<]+%)." "") + (: :gsub "[ \n]%([^<]+%)." "") + ;; code blocks + (: :gsub "

\n?([^<]+)\n?
" "```lua\n%1\n```") + ;; list items to indented * thingies + (: :gsub "
  • ([^<]+)
    • " #(.. "* " ($:gsub "\n" "\n "))) + (: :gsub "" "")) + ;; check to ensure that all the tags have been defeated + tag (str:match "<[^>]+>[^>]+>")] + (when tag (error (.. tag "\n" str))) + (-> str + ;; trim whitespace + (: :match "^%s*(.-)%s*$") + ;; html things + (: :gsub " " " ") + (: :gsub "–" "–") + (: :gsub "—" "—") + ;; For some reason, they use an html middot, but we want to use periods. + (: :gsub "···" "...") + (: :gsub ">" ">") + (: :gsub "<" "<") + (: :gsub "&" "<") + (: :gsub "π" "π") + (: :gsub "\n\n+" "\n\n")))) -(local module-items []) -(local modules []) -(fn loop [prev] - (let [header (stdlib:find "

    .-\n" (+ prev 1)) - section (stdlib:sub prev (if header (- header 1)))] - (let [index (section:find "

    ")] - (if index - (do - (table.insert module-items (section:sub 1 (- index 1))) - (table.insert modules (section:sub index))) - (table.insert module-items section))) - (when header (loop header)))) -(loop (stdlib:find "

    .-\n")) - -;; I should probably split more of this file into functions like this -(fn process-html-thing [html-describing-the-thing] - (let [(header description) (html-describing-the-thing:match "^(.-)\n+(.-)\n*$") +(fn parse-h3-section [html] + "parse a section that starts with an h3 tag. These are individual functions/variables." + (let [(header description) (html:match "^(.-)\n+(.-)\n*$") optional-args [] signature (header:match "(.-)") ;; strip commas @@ -60,38 +107,80 @@ signature (signature:gsub " +" " ") signature (signature:gsub " +%)" ")") - ;; delete

    tags - description (description:gsub "" "") - ;; trim whitespace - description (description:match "^%s*(.-)%s*$") ;; tags for optional args description (accumulate [description description _ arg (ipairs optional-args)] (description:gsub (.. "" arg "") (.. "`?" arg "`"))) - ;; tags for the rest - description (description:gsub "\"(.-)\"" "`\"%1\"`") - description (description:gsub "\'(.-)\'" "`\"%1\"`") - description (description:gsub "(.-)" "`%1`") - description (description:gsub " " " ") - description (description:gsub "–" "–") - description (description:gsub "—" "—") + ;; trim off the string pattern and string.pack/string.unpack format docs + description (description:gsub "\n[^\n]*

    .*" "") + description (html-to-markdown description) name (signature:match "[^() ]+") key (name:match "[^.:]+$") ?module (and (name:find "[.:]") (name:match "^[^.:]+"))] - (assert (not (signature:find "[%[%]]")) (.. "bad signature " signature)) - ;; Debug prints for now. - (when (not ?module) - (print - (.. (view ?module) - (view key) - " " - (view {:binding (signature:match "[^() ]+") - :metadata {:fnl/docstring description}})))))) + (values ?module + key + {:binding (signature:match "[^() ]+") + :metadata {:fnl/docstring description}}))) + +(fn parse-h2-section [html] + "parse a section that starts with an h2 tag. These are the main modules." + (let [(title description) (html:match "(.-)\n(.*)") + module-name (if (title:find "Coroutine") + "coroutine" + (title:find "Modules") + "package" + (title:find "String") + "string" + (title:find "UTF") + "utf8" + (title:find "Mathematical") + "math" + (title:find "Input and Output") + "io" + (title:find "Operating System") + "os" + (title:find "Debug") + "debug" + (title:find "Bitwise") + "bit32" + (title:find "Table") + "table") + description (html-to-markdown description)] + (assert module-name title) + (values module-name + {:binding module-name + :fields {} + :metadata {:fnl/docstring description}}))) -(var done false) -(each [_ section (ipairs module-items) &until done] - (process-html-thing section)) - ; (if (= (io.read) "done") - ; (set done true))) +(fn main [] + (each [_ version (ipairs [:5.1 :5.2 :5.3 :5.4])] + (let [infile (open-file-cached + (.. "lua" version ".html") + (.. "https://www.lua.org/manual/" version "/manual.html")) + (modules module-items) (parse-html (infile:read :a)) + docs + (collect [_ module (ipairs modules)] + (parse-h2-section module))] + + (each [_ section (ipairs module-items)] + (let [(mod k v) (parse-h3-section section)] + (if (not mod) + (tset docs k v) + (not= mod "file") + (do + (assert (. docs mod) (.. mod " not found")) + (tset (. docs mod :fields) k v))))) + + (print (.. "Emitting docs for lua" version)) + (let [outfile (io.open (.. "src/fennel-ls/docs/lua" (version:gsub "%." "") ".fnl") :w)] + (local {: view : sym : list} (require :fennel)) + (outfile:write + (view (list (sym :local) (sym :docs) docs)) + "\n" + "(set docs._G.fields docs)\n" + "docs\n")) + (print (.. "Emitted docs for lua" version))))) + +(main)