diff options
Diffstat (limited to 'Data/Libraries/LDoc/ldoc/parse.lua')
| -rw-r--r-- | Data/Libraries/LDoc/ldoc/parse.lua | 430 |
1 files changed, 430 insertions, 0 deletions
diff --git a/Data/Libraries/LDoc/ldoc/parse.lua b/Data/Libraries/LDoc/ldoc/parse.lua new file mode 100644 index 0000000..11ade60 --- /dev/null +++ b/Data/Libraries/LDoc/ldoc/parse.lua @@ -0,0 +1,430 @@ +-- parsing code for doc comments + +local utils = require 'pl.utils' +local List = require 'pl.List' +-- local Map = require 'pl.Map' +local stringio = require 'pl.stringio' +local lexer = require 'ldoc.lexer' +local tools = require 'ldoc.tools' +local doc = require 'ldoc.doc' +local Item,File = doc.Item,doc.File +local unpack = utils.unpack + +------ Parsing the Source -------------- +-- This uses the lexer from PL, but it should be possible to use Peter Odding's +-- excellent Lpeg based lexer instead. + +local parse = {} + +local tnext, append = lexer.skipws, table.insert + +-- a pattern particular to LuaDoc tag lines: the line must begin with @TAG, +-- followed by the value, which may extend over several lines. +local luadoc_tag = '^%s*@(%w+)' +local luadoc_tag_value = luadoc_tag..'(.*)' +local luadoc_tag_mod_and_value = luadoc_tag..'%[([^%]]*)%](.*)' + +-- assumes that the doc comment consists of distinct tag lines +local function parse_at_tags(text) + local lines = stringio.lines(text) + local preamble, line = tools.grab_while_not(lines,luadoc_tag) + local tag_items = {} + local follows + while line do + local tag, mod_string, rest = line :match(luadoc_tag_mod_and_value) + if not tag then tag, rest = line :match (luadoc_tag_value) end + local modifiers + if mod_string then + modifiers = { } + for x in mod_string :gmatch "[^,]+" do + local k, v = x :match "^([^=]+)=(.*)$" + if not k then k, v = x, true end -- wuz x, x + modifiers[k] = v + end + end + -- follows: end of current tag + -- line: beginning of next tag (for next iteration) + follows, line = tools.grab_while_not(lines,luadoc_tag) + append(tag_items,{tag, rest .. '\n' .. follows, modifiers}) + end + return preamble,tag_items +end + +--local colon_tag = '%s*(%a+):%s' +local colon_tag = '%s*(%S-):%s' +local colon_tag_value = colon_tag..'(.*)' + +local function parse_colon_tags (text) + local lines = stringio.lines(text) + local preamble, line = tools.grab_while_not(lines,colon_tag) + local tag_items, follows = {} + while line do + local tag, rest = line:match(colon_tag_value) + follows, line = tools.grab_while_not(lines,colon_tag) + local value = rest .. '\n' .. follows + if tag:match '^[%?!]' then + tag = tag:gsub('^!','') + value = tag .. ' ' .. value + tag = 'tparam' + end + append(tag_items,{tag, value}) + end + return preamble,tag_items +end + +-- Tags are stored as an ordered multi map from strings to strings +-- If the same key is used, then the value becomes a list +local Tags = {} +Tags.__index = Tags + +function Tags.new (t,name) + local class + if name then + class = t + t = {} + end + t._order = List() + local tags = setmetatable(t,Tags) + if name then + tags:add('class',class) + tags:add('name',name) + end + return tags +end + +function Tags:add (tag,value,modifiers) + if modifiers then -- how modifiers are encoded + value = {value,modifiers=modifiers} + end + local ovalue = self:get(tag) + if ovalue then + ovalue:append(value) + value = ovalue + end + rawset(self,tag,value) + if not ovalue then + self._order:append(tag) + end +end + +function Tags:get (tag) + local ovalue = rawget(self,tag) + if ovalue then -- previous value? + if getmetatable(ovalue) ~= List then + ovalue = List{ovalue} + end + return ovalue + end +end + +function Tags:iter () + return self._order:iter() +end + +local function comment_contains_tags (comment,args) + return (args.colon and comment:find ': ') or (not args.colon and comment:find '@') +end + +-- This takes the collected comment block, and uses the docstyle to +-- extract tags and values. Assume that the summary ends in a period or a question +-- mark, and everything else in the preamble is the description. +-- If a tag appears more than once, then its value becomes a list of strings. +-- Alias substitution and @TYPE NAME shortcutting is handled by Item.check_tag +local function extract_tags (s,args) + local preamble,tag_items + if s:match '^%s*$' then return {} end + if args.colon then --and s:match ':%s' and not s:match '@%a' then + preamble,tag_items = parse_colon_tags(s) + else + preamble,tag_items = parse_at_tags(s) + end + local strip = tools.strip + local summary, description = preamble:match('^(.-[%.?])(%s.+)') + if not summary then + -- perhaps the first sentence did not have a . or ? terminating it. + -- Then try split at linefeed + summary, description = preamble:match('^(.-\n\n)(.+)') + if not summary then + summary = preamble + end + end -- and strip(description) ? + local tags = Tags.new{summary=summary and strip(summary) or '',description=description or ''} + for _,item in ipairs(tag_items) do + local tag, value, modifiers = Item.check_tag(tags,unpack(item)) + -- treat multiline values more gently.. + if not value:match '\n[^\n]+\n' then + value = strip(value) + end + + tags:add(tag,value,modifiers) + end + return tags --Map(tags) +end + + +-- parses a Lua or C file, looking for ldoc comments. These are like LuaDoc comments; +-- they start with multiple '-'. (Block commments are allowed) +-- If they don't define a name tag, then by default +-- it is assumed that a function definition follows. If it is the first comment +-- encountered, then ldoc looks for a call to module() to find the name of the +-- module if there isn't an explicit module name specified. + +local function parse_file(fname, lang, package, args) + local F = File(fname) + local module_found, first_comment = false,true + local current_item, module_item + + F.args = args + F.lang = lang + F.base = package + + local tok,f = lang.lexer(fname) + if not tok then return nil end + + local function lineno () + return tok:lineno() + end + + function F:warning (msg,kind,line) + line = line or lineno() + Item.had_warning = true + io.stderr:write(fname..':'..line..': '..msg,'\n') + end + + function F:error (msg) + self:warning(msg,'error') + io.stderr:write('LDoc error\n') + os.exit(1) + end + + local function add_module(tags,module_found,old_style) + tags:add('name',module_found) + tags:add('class','module') + local item = F:new_item(tags,lineno()) + item.old_style = old_style + module_item = item + end + + local mod + local t,v = tnext(tok) + -- with some coding styles first comment is standard boilerplate; option to ignore this. + if args.boilerplate and t == 'comment' then + -- hack to deal with boilerplate inside Lua block comments + if v:match '%s*%-%-%[%[' then lang:grab_block_comment(v,tok) end + t,v = tnext(tok) + end + if t == '#' then -- skip Lua shebang line, if present + while t and t ~= 'comment' do t,v = tnext(tok) end + if t == nil then + F:warning('empty file') + return nil + end + end + if lang.parse_module_call and t ~= 'comment' then + local prev_token + while t do + if prev_token ~= '.' and prev_token ~= ':' and t == 'iden' and v == 'module' then + break + end + prev_token = t + t, v = tnext(tok) + end + if not t then + if not args.ignore then + F:warning("no module() call found; no initial doc comment") + end + --return nil + else + mod,t,v = lang:parse_module_call(tok,t,v) + if mod and mod ~= '...' then + add_module(Tags.new{summary='(no description)'},mod,true) + first_comment = false + module_found = true + end + end + end + local ok, err = xpcall(function() + while t do + if t == 'comment' then + local comment = {} + local ldoc_comment,block = lang:start_comment(v) + + if ldoc_comment and block then + t,v = lang:grab_block_comment(v,tok) + end + + if lang:empty_comment(v) then -- ignore rest of empty start comments + t,v = tok() + if t == 'space' and not v:match '\n' then + t,v = tok() + end + end + + while t and t == 'comment' do + v = lang:trim_comment(v) + append(comment,v) + t,v = tok() + if t == 'space' and not v:match '\n' then + t,v = tok() + end + end + + if t == 'space' then t,v = tnext(tok) end + + local item_follows, tags, is_local, case, parse_error + if ldoc_comment then + comment = table.concat(comment) + if comment:match '^%s*$' then + ldoc_comment = nil + end + end + if ldoc_comment then + if first_comment then + first_comment = false + else + item_follows, is_local, case = lang:item_follows(t,v,tok) + if not item_follows then + parse_error = is_local + is_local = false + end + end + + if item_follows or comment_contains_tags(comment,args) then + tags = extract_tags(comment,args) + + -- explicitly named @module (which is recommended) + if doc.project_level(tags.class) then + module_found = tags.name + -- might be a module returning a single function! + if tags.param or tags['return'] then + local parms, ret = tags.param, tags['return'] + local name = tags.name + tags.param = nil + tags['return'] = nil + tags['class'] = nil + tags['name'] = nil + add_module(tags,name,false) + tags = { + summary = '', + name = 'returns...', + class = 'function', + ['return'] = ret, + param = parms + } + end + end + doc.expand_annotation_item(tags,current_item) + -- if the item has an explicit name or defined meaning + -- then don't continue to do any code analysis! + -- Watch out for the case where there are field or param tags + -- but no class, since these will be fixed up later as module/class + -- entities + if (tags.field or tags.param) and not tags.class then + parse_error = false + end + if tags.name then + if not tags.class then + F:warning("no type specified, assuming function: '"..tags.name.."'") + tags:add('class','function') + end + item_follows, is_local, parse_error = false, false, false + elseif args.no_args_infer then + F:error("No name and type provided (no_args_infer)") + elseif lang:is_module_modifier (tags) then + if not item_follows then + F:warning("@usage or @export followed by unknown code") + break + end + item_follows(tags,tok) + local res, value, tagname = lang:parse_module_modifier(tags,tok,F) + if not res then F:warning(value); break + else + if tagname then + module_item:set_tag(tagname,value) + end + -- don't continue to make an item! + ldoc_comment = false + end + end + end + if parse_error then + F:warning('definition cannot be parsed - '..parse_error) + end + end + -- some hackery necessary to find the module() call + if not module_found and ldoc_comment then + local old_style + module_found,t,v = lang:find_module(tok,t,v) + -- right, we can add the module object ... + old_style = module_found ~= nil + if not module_found or module_found == '...' then + -- we have to guess the module name + module_found = tools.this_module_name(package,fname) + end + if not tags then tags = extract_tags(comment,args) end + add_module(tags,module_found,old_style) + tags = nil + if not t then + F:warning('contains no items','warning',1) + break; + end -- run out of file! + -- if we did bump into a doc comment, then we can continue parsing it + end + + -- end of a block of document comments + if ldoc_comment and tags then + local line = lineno() + if t ~= nil then + if item_follows then -- parse the item definition + local err = item_follows(tags,tok) + if err then F:error(err) end + elseif parse_error then + F:warning('definition cannot be parsed - '..parse_error) + else + lang:parse_extra(tags,tok,case) + end + end + if is_local or tags['local'] then + tags:add('local',true) + end + -- support for standalone fields/properties of classes/modules + if (tags.field or tags.param) and not tags.class then + -- the hack is to take a subfield and pull out its name, + -- (see Tag:add above) but let the subfield itself go through + -- with any modifiers. + local fp = tags.field or tags.param + if type(fp) == 'table' then fp = fp[1] end + fp = tools.extract_identifier(fp) + tags:add('name',fp) + tags:add('class','field') + end + if tags.name then + current_item = F:new_item(tags,line) + current_item.inferred = item_follows ~= nil + if doc.project_level(tags.class) then + if module_item then + F:error("Module already declared!") + end + module_item = current_item + end + end + if not t then break end + end + end + if t ~= 'comment' then t,v = tok() end + end + end,debug.traceback) + if not ok then return F, err end + if f then f:close() end + return F +end + +function parse.file(name,lang, args) + local F,err = parse_file(name,lang,args.package,args) + if err or not F then return F,err end + local ok,err = xpcall(function() F:finish() end,debug.traceback) + if not ok then return F,err end + return F +end + +return parse |