More doc-making work! it works!
This commit is contained in:
XeroOl
parent
bb420adbf9
commit
1c925ea1fb
203
make-docs.fnl
203
make-docs.fnl
@ -1,39 +1,86 @@
|
||||
"work in progress script to generate /src/fennel-ls/docs/lua54.fnl automatically"
|
||||
(local {: view} (require :fennel))
|
||||
(local version "5.4")
|
||||
(local filename (.. "lua" version ".html"))
|
||||
(var file (io.open filename :r))
|
||||
(when (not file)
|
||||
;; I think the repos also have the html file (or the ability to generate it, depending on lua version)
|
||||
;; but for simplicity I'm just grabbing the html straight from the website
|
||||
(os.execute (.. "curl https://www.lua.org/manual/" version "/manual.html > lua" version ".html"))
|
||||
(set file (io.open filename :r)))
|
||||
"Script to generate /src/fennel-ls/docs/lua54.fnl and friends automatically"
|
||||
|
||||
(local html (file:read :a))
|
||||
(fn open-file-cached [filename url]
|
||||
(var file (io.open filename :r))
|
||||
(when (not file)
|
||||
;; I think the repos also have the html file (or the ability to generate it, depending on lua version)
|
||||
;; but for simplicity I'm just grabbing the html straight from the website
|
||||
(print (.. "Downloading " url))
|
||||
(os.execute (.. "curl " url " > " filename))
|
||||
(set file (io.open filename :r)))
|
||||
file)
|
||||
|
||||
;; The section of the html about the standard library
|
||||
(local begin-index (assert (html:find "<h2>.%.1 – <a name=\".%.1\">Basic Functions.-\n")))
|
||||
(local end-index (assert (html:find "<h1>. – <a name=\".\".-\n" begin-index)))
|
||||
(local stdlib (html:sub begin-index (- end-index 1)))
|
||||
(fn parse-html [html]
|
||||
"splits the lua manual into the relevant sections"
|
||||
(let [begin-index (assert (html:find "<h2>.%.1 – <a name=\".%.1\">Basic Functions.-\n") "no basic functions?")
|
||||
end-index (assert (html:find "<h1>. – <a name=\".\".-\n" begin-index))
|
||||
stdlib (html:sub begin-index (- end-index 1))
|
||||
fields []
|
||||
modules []]
|
||||
(fn loop [prev]
|
||||
(let [header (stdlib:find "<hr><h3>.-\n" (+ prev 1))
|
||||
section (stdlib:sub prev (if header (- header 1)))]
|
||||
(let [index (section:find "<h2>")]
|
||||
(if index
|
||||
(do
|
||||
(table.insert fields (section:sub 1 (- index 1)))
|
||||
(table.insert modules (section:sub index)))
|
||||
(table.insert fields section)))
|
||||
(when header (loop header))))
|
||||
(loop (stdlib:find "<hr><h3>.-\n"))
|
||||
(values modules fields)))
|
||||
|
||||
(fn html-to-markdown [str]
|
||||
(let [str
|
||||
(-> str
|
||||
;; delete <p> tags
|
||||
(: :gsub "</?p>" "")
|
||||
;; <code> tags for the rest
|
||||
(: :gsub "\"<code>(.-)</code>\"" "`\"%1\"`")
|
||||
(: :gsub "\'<code>(.-)</code>\'" "`\"%1\"`")
|
||||
(: :gsub "<code>(.-)</code>" "`%1`")
|
||||
(: :gsub "<sup>x</sup>" "ˣ")
|
||||
(: :gsub "<sup>e</sup>" "ᵉ")
|
||||
(: :gsub "<sup>y</sup>" "ʸ")
|
||||
(: :gsub "<sup>51</sup>" "⁵¹")
|
||||
(: :gsub "<sup>32</sup>" "³²")
|
||||
; ᵃᵇᶜᵈᵉᶠᵍʰⁱʲᵏˡᵐⁿᵒᵖ𐞥ʳˢᵗᵘᵛʷˣʸᶻ
|
||||
;; bold to **
|
||||
(: :gsub "<em>([^<]+)</em>" "*%1*")
|
||||
(: :gsub "<b>([^<]+)</b>" "**%1**")
|
||||
;; defeat all the links
|
||||
(: :gsub "<a name=\"pdf%-[^\"]+\">([^<]+)</a>" "%1")
|
||||
(: :gsub "<a href=\"#pdf%-[^\"]+\">([^<]+)</a>" "%1")
|
||||
(: :gsub "<a href=\"#lua_[^\"]+\">([^<]+)</a>" "%1")
|
||||
(: :gsub "See <a href=\"#[^\"]+\">[^<]+</a>[^%.]+%." "")
|
||||
(: :gsub "[ \n]%(see <a href=\"#[^\"]+\">[^<]+</a>%)." "")
|
||||
(: :gsub "[ \n]%(<a href=\"#[^\"]+\">[^<]+</a>%)." "")
|
||||
;; code blocks
|
||||
(: :gsub "<pre>\n?([^<]+)\n?</pre>" "```lua\n%1\n```")
|
||||
;; list items to indented * thingies
|
||||
(: :gsub "<li>([^<]+)</li>" #(.. "* " ($:gsub "\n" "\n ")))
|
||||
(: :gsub "</?ul>" ""))
|
||||
;; check to ensure that all the tags have been defeated
|
||||
tag (str:match "<[^>]+>[^>]+>")]
|
||||
(when tag (error (.. tag "\n" str)))
|
||||
(-> str
|
||||
;; trim whitespace
|
||||
(: :match "^%s*(.-)%s*$")
|
||||
;; html things
|
||||
(: :gsub " " " ")
|
||||
(: :gsub "–" "–")
|
||||
(: :gsub "—" "—")
|
||||
;; For some reason, they use an html middot, but we want to use periods.
|
||||
(: :gsub "···" "...")
|
||||
(: :gsub ">" ">")
|
||||
(: :gsub "<" "<")
|
||||
(: :gsub "&" "<")
|
||||
(: :gsub "π" "π")
|
||||
(: :gsub "\n\n+" "\n\n"))))
|
||||
|
||||
(local module-items [])
|
||||
(local modules [])
|
||||
(fn loop [prev]
|
||||
(let [header (stdlib:find "<hr><h3>.-\n" (+ prev 1))
|
||||
section (stdlib:sub prev (if header (- header 1)))]
|
||||
(let [index (section:find "<h2>")]
|
||||
(if index
|
||||
(do
|
||||
(table.insert module-items (section:sub 1 (- index 1)))
|
||||
(table.insert modules (section:sub index)))
|
||||
(table.insert module-items section)))
|
||||
(when header (loop header))))
|
||||
(loop (stdlib:find "<hr><h3>.-\n"))
|
||||
|
||||
;; I should probably split more of this file into functions like this
|
||||
(fn process-html-thing [html-describing-the-thing]
|
||||
(let [(header description) (html-describing-the-thing:match "^(.-)\n+(.-)\n*$")
|
||||
(fn parse-h3-section [html]
|
||||
"parse a section that starts with an h3 tag. These are individual functions/variables."
|
||||
(let [(header description) (html:match "^(.-)\n+(.-)\n*$")
|
||||
optional-args []
|
||||
signature (header:match "<code>(.-)</code>")
|
||||
;; strip commas
|
||||
@ -60,38 +107,80 @@
|
||||
signature (signature:gsub " +" " ")
|
||||
signature (signature:gsub " +%)" ")")
|
||||
|
||||
;; delete <p> tags
|
||||
description (description:gsub "</?p>" "")
|
||||
;; trim whitespace
|
||||
description (description:match "^%s*(.-)%s*$")
|
||||
;; <code> tags for optional args
|
||||
description (accumulate [description description _ arg (ipairs optional-args)]
|
||||
(description:gsub (.. "<code>" arg "</code>")
|
||||
(.. "`?" arg "`")))
|
||||
;; <code> tags for the rest
|
||||
description (description:gsub "\"<code>(.-)</code>\"" "`\"%1\"`")
|
||||
description (description:gsub "\'<code>(.-)</code>\'" "`\"%1\"`")
|
||||
description (description:gsub "<code>(.-)</code>" "`%1`")
|
||||
description (description:gsub " " " ")
|
||||
description (description:gsub "–" "–")
|
||||
description (description:gsub "—" "—")
|
||||
;; trim off the string pattern and string.pack/string.unpack format docs
|
||||
description (description:gsub "\n[^\n]*<h3>.*" "")
|
||||
description (html-to-markdown description)
|
||||
name (signature:match "[^() ]+")
|
||||
key (name:match "[^.:]+$")
|
||||
?module (and (name:find "[.:]") (name:match "^[^.:]+"))]
|
||||
|
||||
(assert (not (signature:find "[%[%]]")) (.. "bad signature " signature))
|
||||
;; Debug prints for now.
|
||||
(when (not ?module)
|
||||
(print
|
||||
(.. (view ?module)
|
||||
(view key)
|
||||
" "
|
||||
(view {:binding (signature:match "[^() ]+")
|
||||
:metadata {:fnl/docstring description}}))))))
|
||||
(values ?module
|
||||
key
|
||||
{:binding (signature:match "[^() ]+")
|
||||
:metadata {:fnl/docstring description}})))
|
||||
|
||||
(fn parse-h2-section [html]
|
||||
"parse a section that starts with an h2 tag. These are the main modules."
|
||||
(let [(title description) (html:match "(.-)\n(.*)")
|
||||
module-name (if (title:find "Coroutine")
|
||||
"coroutine"
|
||||
(title:find "Modules")
|
||||
"package"
|
||||
(title:find "String")
|
||||
"string"
|
||||
(title:find "UTF")
|
||||
"utf8"
|
||||
(title:find "Mathematical")
|
||||
"math"
|
||||
(title:find "Input and Output")
|
||||
"io"
|
||||
(title:find "Operating System")
|
||||
"os"
|
||||
(title:find "Debug")
|
||||
"debug"
|
||||
(title:find "Bitwise")
|
||||
"bit32"
|
||||
(title:find "Table")
|
||||
"table")
|
||||
description (html-to-markdown description)]
|
||||
(assert module-name title)
|
||||
(values module-name
|
||||
{:binding module-name
|
||||
:fields {}
|
||||
:metadata {:fnl/docstring description}})))
|
||||
|
||||
|
||||
(var done false)
|
||||
(each [_ section (ipairs module-items) &until done]
|
||||
(process-html-thing section))
|
||||
; (if (= (io.read) "done")
|
||||
; (set done true)))
|
||||
(fn main []
|
||||
(each [_ version (ipairs [:5.1 :5.2 :5.3 :5.4])]
|
||||
(let [infile (open-file-cached
|
||||
(.. "lua" version ".html")
|
||||
(.. "https://www.lua.org/manual/" version "/manual.html"))
|
||||
(modules module-items) (parse-html (infile:read :a))
|
||||
docs
|
||||
(collect [_ module (ipairs modules)]
|
||||
(parse-h2-section module))]
|
||||
|
||||
(each [_ section (ipairs module-items)]
|
||||
(let [(mod k v) (parse-h3-section section)]
|
||||
(if (not mod)
|
||||
(tset docs k v)
|
||||
(not= mod "file")
|
||||
(do
|
||||
(assert (. docs mod) (.. mod " not found"))
|
||||
(tset (. docs mod :fields) k v)))))
|
||||
|
||||
(print (.. "Emitting docs for lua" version))
|
||||
(let [outfile (io.open (.. "src/fennel-ls/docs/lua" (version:gsub "%." "") ".fnl") :w)]
|
||||
(local {: view : sym : list} (require :fennel))
|
||||
(outfile:write
|
||||
(view (list (sym :local) (sym :docs) docs))
|
||||
"\n"
|
||||
"(set docs._G.fields docs)\n"
|
||||
"docs\n"))
|
||||
(print (.. "Emitted docs for lua" version)))))
|
||||
|
||||
(main)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user