diff options
| author | chai <chaifix@163.com> | 2021-10-30 11:32:16 +0800 |
|---|---|---|
| committer | chai <chaifix@163.com> | 2021-10-30 11:32:16 +0800 |
| commit | 42ec7286b2d36a9ba22925f816a17cb1cc2aa5ce (patch) | |
| tree | 24bc7009457a8d7500f264e89946dc20d069294f /Data/Libraries/Penlight/docs/libraries/pl.lexer.html | |
| parent | 164885fd98d48703bd771f802d79557b7db97431 (diff) | |
+ Penlight
Diffstat (limited to 'Data/Libraries/Penlight/docs/libraries/pl.lexer.html')
| -rw-r--r-- | Data/Libraries/Penlight/docs/libraries/pl.lexer.html | 524 |
1 files changed, 524 insertions, 0 deletions
diff --git a/Data/Libraries/Penlight/docs/libraries/pl.lexer.html b/Data/Libraries/Penlight/docs/libraries/pl.lexer.html new file mode 100644 index 0000000..7b3d6fd --- /dev/null +++ b/Data/Libraries/Penlight/docs/libraries/pl.lexer.html @@ -0,0 +1,524 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> +<html> +<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/> +<head> + <title>Penlight Documentation</title> + <link rel="stylesheet" href="../ldoc_fixed.css" type="text/css" /> +</head> +<body> + +<div id="container"> + +<div id="product"> + <div id="product_logo"></div> + <div id="product_name"><big><b></b></big></div> + <div id="product_description"></div> +</div> <!-- id="product" --> + + +<div id="main"> + + +<!-- Menu --> + +<div id="navigation"> +<br/> +<h1>Penlight</h1> + +<ul> + <li><a href="https://github.com/lunarmodules/Penlight">GitHub Project</a></li> + <li><a href="../index.html">Documentation</a></li> +</ul> + +<h2>Contents</h2> +<ul> +<li><a href="#Functions">Functions</a></li> +</ul> + + +<h2>Libraries</h2> +<ul class="nowrap"> + <li><a href="../libraries/pl.html">pl</a></li> + <li><a href="../libraries/pl.app.html">pl.app</a></li> + <li><a href="../libraries/pl.array2d.html">pl.array2d</a></li> + <li><a href="../libraries/pl.class.html">pl.class</a></li> + <li><a href="../libraries/pl.compat.html">pl.compat</a></li> + <li><a href="../libraries/pl.comprehension.html">pl.comprehension</a></li> + <li><a href="../libraries/pl.config.html">pl.config</a></li> + <li><a href="../libraries/pl.data.html">pl.data</a></li> + <li><a href="../libraries/pl.dir.html">pl.dir</a></li> + <li><a href="../libraries/pl.file.html">pl.file</a></li> + <li><a href="../libraries/pl.func.html">pl.func</a></li> + <li><a href="../libraries/pl.import_into.html">pl.import_into</a></li> + <li><a href="../libraries/pl.input.html">pl.input</a></li> + <li><a href="../libraries/pl.lapp.html">pl.lapp</a></li> + <li><strong>pl.lexer</strong></li> + <li><a href="../libraries/pl.luabalanced.html">pl.luabalanced</a></li> + <li><a href="../libraries/pl.operator.html">pl.operator</a></li> + <li><a href="../libraries/pl.path.html">pl.path</a></li> + <li><a href="../libraries/pl.permute.html">pl.permute</a></li> + <li><a href="../libraries/pl.pretty.html">pl.pretty</a></li> + <li><a href="../libraries/pl.seq.html">pl.seq</a></li> + <li><a href="../libraries/pl.sip.html">pl.sip</a></li> + <li><a href="../libraries/pl.strict.html">pl.strict</a></li> + <li><a href="../libraries/pl.stringio.html">pl.stringio</a></li> + <li><a href="../libraries/pl.stringx.html">pl.stringx</a></li> + <li><a href="../libraries/pl.tablex.html">pl.tablex</a></li> + <li><a href="../libraries/pl.template.html">pl.template</a></li> + <li><a href="../libraries/pl.test.html">pl.test</a></li> + <li><a href="../libraries/pl.text.html">pl.text</a></li> + <li><a href="../libraries/pl.types.html">pl.types</a></li> + <li><a href="../libraries/pl.url.html">pl.url</a></li> + <li><a href="../libraries/pl.utils.html">pl.utils</a></li> + <li><a href="../libraries/pl.xml.html">pl.xml</a></li> +</ul> +<h2>Classes</h2> +<ul class="nowrap"> + <li><a href="../classes/pl.Date.html">pl.Date</a></li> + <li><a href="../classes/pl.List.html">pl.List</a></li> + <li><a href="../classes/pl.Map.html">pl.Map</a></li> + <li><a href="../classes/pl.MultiMap.html">pl.MultiMap</a></li> + <li><a href="../classes/pl.OrderedMap.html">pl.OrderedMap</a></li> + <li><a href="../classes/pl.Set.html">pl.Set</a></li> +</ul> +<h2>Manual</h2> +<ul class="nowrap"> + <li><a href="../manual/01-introduction.md.html">Introduction</a></li> + <li><a href="../manual/02-arrays.md.html">Tables and Arrays</a></li> + <li><a href="../manual/03-strings.md.html">Strings. Higher-level operations on strings.</a></li> + <li><a href="../manual/04-paths.md.html">Paths and Directories</a></li> + <li><a href="../manual/05-dates.md.html">Date and Time</a></li> + <li><a href="../manual/06-data.md.html">Data</a></li> + <li><a href="../manual/07-functional.md.html">Functional Programming</a></li> + <li><a href="../manual/08-additional.md.html">Additional Libraries</a></li> + <li><a href="../manual/09-discussion.md.html">Technical Choices</a></li> +</ul> +<h2>Examples</h2> +<ul class="nowrap"> + <li><a href="../examples/seesubst.lua.html">seesubst.lua</a></li> + <li><a href="../examples/sipscan.lua.html">sipscan.lua</a></li> + <li><a href="../examples/symbols.lua.html">symbols.lua</a></li> + <li><a href="../examples/test-cmp.lua.html">test-cmp.lua</a></li> + <li><a href="../examples/test-data.lua.html">test-data.lua</a></li> + <li><a href="../examples/test-listcallbacks.lua.html">test-listcallbacks.lua</a></li> + <li><a href="../examples/test-pretty.lua.html">test-pretty.lua</a></li> + <li><a href="../examples/test-symbols.lua.html">test-symbols.lua</a></li> + <li><a href="../examples/testclone.lua.html">testclone.lua</a></li> + <li><a href="../examples/testconfig.lua.html">testconfig.lua</a></li> + <li><a href="../examples/testglobal.lua.html">testglobal.lua</a></li> + <li><a href="../examples/testinputfields.lua.html">testinputfields.lua</a></li> + <li><a href="../examples/testinputfields2.lua.html">testinputfields2.lua</a></li> + <li><a href="../examples/testxml.lua.html">testxml.lua</a></li> + <li><a href="../examples/which.lua.html">which.lua</a></li> +</ul> + +</div> + +<div id="content"> + +<h1>Module <code>pl.lexer</code></h1> +<p>Lexical scanner for creating a sequence of tokens from text.</p> +<p> <code>lexer.scan(s)</code> returns an iterator over all tokens found in the + string <code>s</code>. This iterator returns two values, a token type string + (such as 'string' for quoted string, 'iden' for identifier) and the value of the + token.</p> + +<p> Versions specialized for Lua and C are available; these also handle block comments + and classify keywords as 'keyword' tokens. For example:</p> + + +<pre> +> s = <span class="string">'for i=1,n do'</span> +> <span class="keyword">for</span> t,v <span class="keyword">in</span> lexer.lua(s) <span class="keyword">do</span> <span class="global">print</span>(t,v) <span class="keyword">end</span> +keyword <span class="keyword">for</span> +iden i += = +number <span class="number">1</span> +, , +iden n +keyword <span class="keyword">do</span> +</pre> + +<p> See the Guide for further <a href="../manual/06-data.md.html#Lexical_Scanning">discussion</a></p> + + +<h2><a href="#Functions">Functions</a></h2> +<table class="function_list"> + <tr> + <td class="name" nowrap><a href="#scan">scan (s, matches[, filter[, options]])</a></td> + <td class="summary">create a plain token iterator from a string or file-like object.</td> + </tr> + <tr> + <td class="name" nowrap><a href="#insert">insert (tok, a1, a2)</a></td> + <td class="summary">insert tokens into a stream.</td> + </tr> + <tr> + <td class="name" nowrap><a href="#getline">getline (tok)</a></td> + <td class="summary">get everything in a stream upto a newline.</td> + </tr> + <tr> + <td class="name" nowrap><a href="#lineno">lineno (tok)</a></td> + <td class="summary">get current line number.</td> + </tr> + <tr> + <td class="name" nowrap><a href="#getrest">getrest (tok)</a></td> + <td class="summary">get the rest of the stream.</td> + </tr> + <tr> + <td class="name" nowrap><a href="#get_keywords">get_keywords ()</a></td> + <td class="summary">get the Lua keywords as a set-like table.</td> + </tr> + <tr> + <td class="name" nowrap><a href="#lua">lua (s[, filter[, options]])</a></td> + <td class="summary">create a Lua token iterator from a string or file-like object.</td> + </tr> + <tr> + <td class="name" nowrap><a href="#cpp">cpp (s[, filter[, options]])</a></td> + <td class="summary">create a C/C++ token iterator from a string or file-like object.</td> + </tr> + <tr> + <td class="name" nowrap><a href="#get_separated_list">get_separated_list (tok[, endtoken=')'[, delim=']])</a></td> + <td class="summary">get a list of parameters separated by a delimiter from a stream.</td> + </tr> + <tr> + <td class="name" nowrap><a href="#skipws">skipws (tok)</a></td> + <td class="summary">get the next non-space token from the stream.</td> + </tr> + <tr> + <td class="name" nowrap><a href="#expecting">expecting (tok, expected_type, no_skip_ws)</a></td> + <td class="summary">get the next token, which must be of the expected type.</td> + </tr> +</table> + +<br/> +<br/> + + + <h2 class="section-header "><a name="Functions"></a>Functions</h2> + + <dl class="function"> + <dt> + <a name = "scan"></a> + <strong>scan (s, matches[, filter[, options]])</strong> + </dt> + <dd> + create a plain token iterator from a string or file-like object. + + + <h3>Parameters:</h3> + <ul> + <li><span class="parameter">s</span> + <span class="types"><a class="type" href="https://www.lua.org/manual/5.1/manual.html#5.4">string</a> or <span class="type">file</span></span> + a string or a file-like object with <code>:read()</code> method returning lines. + </li> + <li><span class="parameter">matches</span> + <span class="types"><span class="type">tab</span></span> + an optional match table - array of token descriptions. + A token is described by a <code>{pattern, action}</code> pair, where <code>pattern</code> should match + token body and <code>action</code> is a function called when a token of described type is found. + </li> + <li><span class="parameter">filter</span> + <span class="types"><span class="type">tab</span></span> + a table of token types to exclude, by default <code>{space=true}</code> + (<em>optional</em>) + </li> + <li><span class="parameter">options</span> + <span class="types"><span class="type">tab</span></span> + a table of options; by default, <code>{number=true,string=true}</code>, + which means convert numbers and strip string quotes. + (<em>optional</em>) + </li> + </ul> + + + + + +</dd> + <dt> + <a name = "insert"></a> + <strong>insert (tok, a1, a2)</strong> + </dt> + <dd> + insert tokens into a stream. + + + <h3>Parameters:</h3> + <ul> + <li><span class="parameter">tok</span> + a token stream + </li> + <li><span class="parameter">a1</span> + a string is the type, a table is a token list and + a function is assumed to be a token-like iterator (returns type & value) + </li> + <li><span class="parameter">a2</span> + <span class="types"><a class="type" href="https://www.lua.org/manual/5.1/manual.html#5.4">string</a></span> + a string is the value + </li> + </ul> + + + + + +</dd> + <dt> + <a name = "getline"></a> + <strong>getline (tok)</strong> + </dt> + <dd> + get everything in a stream upto a newline. + + + <h3>Parameters:</h3> + <ul> + <li><span class="parameter">tok</span> + a token stream + </li> + </ul> + + <h3>Returns:</h3> + <ol> + + a string + </ol> + + + + +</dd> + <dt> + <a name = "lineno"></a> + <strong>lineno (tok)</strong> + </dt> + <dd> + get current line number. + + + <h3>Parameters:</h3> + <ul> + <li><span class="parameter">tok</span> + a token stream + </li> + </ul> + + <h3>Returns:</h3> + <ol> + + the line number. + if the input source is a file-like object, + also return the column. + </ol> + + + + +</dd> + <dt> + <a name = "getrest"></a> + <strong>getrest (tok)</strong> + </dt> + <dd> + get the rest of the stream. + + + <h3>Parameters:</h3> + <ul> + <li><span class="parameter">tok</span> + a token stream + </li> + </ul> + + <h3>Returns:</h3> + <ol> + + a string + </ol> + + + + +</dd> + <dt> + <a name = "get_keywords"></a> + <strong>get_keywords ()</strong> + </dt> + <dd> + get the Lua keywords as a set-like table. + So <code>res["and"]</code> etc would be <code>true</code>. + + + + <h3>Returns:</h3> + <ol> + + a table + </ol> + + + + +</dd> + <dt> + <a name = "lua"></a> + <strong>lua (s[, filter[, options]])</strong> + </dt> + <dd> + create a Lua token iterator from a string or file-like object. + Will return the token type and value. + + + <h3>Parameters:</h3> + <ul> + <li><span class="parameter">s</span> + <span class="types"><a class="type" href="https://www.lua.org/manual/5.1/manual.html#5.4">string</a></span> + the string + </li> + <li><span class="parameter">filter</span> + <span class="types"><span class="type">tab</span></span> + a table of token types to exclude, by default <code>{space=true,comments=true}</code> + (<em>optional</em>) + </li> + <li><span class="parameter">options</span> + <span class="types"><span class="type">tab</span></span> + a table of options; by default, <code>{number=true,string=true}</code>, + which means convert numbers and strip string quotes. + (<em>optional</em>) + </li> + </ul> + + + + + +</dd> + <dt> + <a name = "cpp"></a> + <strong>cpp (s[, filter[, options]])</strong> + </dt> + <dd> + create a C/C++ token iterator from a string or file-like object. + Will return the token type type and value. + + + <h3>Parameters:</h3> + <ul> + <li><span class="parameter">s</span> + <span class="types"><a class="type" href="https://www.lua.org/manual/5.1/manual.html#5.4">string</a></span> + the string + </li> + <li><span class="parameter">filter</span> + <span class="types"><span class="type">tab</span></span> + a table of token types to exclude, by default <code>{space=true,comments=true}</code> + (<em>optional</em>) + </li> + <li><span class="parameter">options</span> + <span class="types"><span class="type">tab</span></span> + a table of options; by default, <code>{number=true,string=true}</code>, + which means convert numbers and strip string quotes. + (<em>optional</em>) + </li> + </ul> + + + + + +</dd> + <dt> + <a name = "get_separated_list"></a> + <strong>get_separated_list (tok[, endtoken=')'[, delim=']])</strong> + </dt> + <dd> + get a list of parameters separated by a delimiter from a stream. + + + <h3>Parameters:</h3> + <ul> + <li><span class="parameter">tok</span> + the token stream + </li> + <li><span class="parameter">endtoken</span> + <span class="types"><a class="type" href="https://www.lua.org/manual/5.1/manual.html#5.4">string</a></span> + end of list. Can be '\n' + (<em>default</em> ')') + </li> + <li><span class="parameter">delim</span> + <span class="types"><a class="type" href="https://www.lua.org/manual/5.1/manual.html#5.4">string</a></span> + separator + (<em>default</em> ') + </li> + </ul> + + <h3>Returns:</h3> + <ol> + + a list of token lists. + </ol> + + + + +</dd> + <dt> + <a name = "skipws"></a> + <strong>skipws (tok)</strong> + </dt> + <dd> + get the next non-space token from the stream. + + + <h3>Parameters:</h3> + <ul> + <li><span class="parameter">tok</span> + the token stream. + </li> + </ul> + + + + + +</dd> + <dt> + <a name = "expecting"></a> + <strong>expecting (tok, expected_type, no_skip_ws)</strong> + </dt> + <dd> + get the next token, which must be of the expected type. + Throws an error if this type does not match! + + + <h3>Parameters:</h3> + <ul> + <li><span class="parameter">tok</span> + the token stream + </li> + <li><span class="parameter">expected_type</span> + <span class="types"><a class="type" href="https://www.lua.org/manual/5.1/manual.html#5.4">string</a></span> + the token type + </li> + <li><span class="parameter">no_skip_ws</span> + <span class="types"><span class="type">bool</span></span> + whether we should skip whitespace + </li> + </ul> + + + + + +</dd> +</dl> + + +</div> <!-- id="content" --> +</div> <!-- id="main" --> +<div id="about"> +<i>generated by <a href="http://github.com/stevedonovan/LDoc">LDoc 1.4.6</a></i> +</div> <!-- id="about" --> +</div> <!-- id="container" --> +</body> +</html> |