enhance(template): Preserve line numbers (#468)

CHANGELOG.md

@@ -15,6 +15,8 @@ see [CONTRIBUTING.md](CONTRIBUTING.md#release-instructions-for-a-new-version) fo
    [#447](https://github.com/lunarmodules/Penlight/pull/447)
  - fix(template) using `%` as an escape character caused the expression to not be recognized
    [#452](https://github.com/lunarmodules/Penlight/pull/452)
+ - enhance(template): Preserve line numbers
+   [#468](https://github.com/lunarmodules/Penlight/pull/468)
 
 ## 1.13.1 (2022-Jul-22)
  - fix: `warn` unquoted argument

lua/pl/template.lua

@@ -30,63 +30,72 @@
 
 local utils = require 'pl.utils'
 
-local append,format,strsub,strfind,strgsub = table.insert,string.format,string.sub,string.find,string.gsub
+local append, concat = table.insert, table.concat
+local format, strsub, strfind, strgsub, strrep = string.format, string.sub, string.find, string.gsub, string.rep
 
-local APPENDER = "\n__R_size = __R_size + 1; __R_table[__R_size] = "
+local APPENDER = " __R_size = __R_size + 1; __R_table[__R_size] = "
 
+-- When this function returns, `pieces` is guaranteed to hold a complete Lua
+-- statement, meaning that new statements can be appended without creating
+-- invalid Lua code.
 local function parseDollarParen(pieces, chunk, exec_pat, newline)
     local s = 1
     for term, executed, e in chunk:gmatch(exec_pat) do
-        executed = '('..strsub(executed,2,-2)..')'
-        append(pieces, APPENDER..format("%q", strsub(chunk,s, term - 1)))
-        append(pieces, APPENDER..format("__tostring(%s or '')", executed))
+        executed = '(' .. strsub(executed, 2, -2) .. ')'
+        append(pieces, APPENDER .. format("%q;", strsub(chunk, s, term - 1)))
+        append(pieces, APPENDER .. format("__tostring(%s or '');", executed))
         s = e
     end
-    local r
+    local remainder, newlines_removed
     if newline then
-        r = format("%q", strgsub(strsub(chunk,s),"\n",""))
+        remainder, newlines_removed = strgsub(strsub(chunk, s), "\n", "")
     else
-        r = format("%q", strsub(chunk,s))
+        remainder, newlines_removed = strsub(chunk, s), 0
     end
-    if r ~= '""' then
-        append(pieces, APPENDER..r)
+    if remainder ~= "" then
+        append(pieces, APPENDER .. format("%q;", remainder))
+    end
+    if newlines_removed > 0 then
+        append(pieces, strrep("\n", newlines_removed))
     end
 end
 
-local function parseHashLines(chunk,inline_escape,brackets,esc,newline)
+local function parseHashLines(chunk, inline_escape, brackets, esc, newline)
     -- Escape special characters to avoid invalid expressions
     inline_escape = utils.escape(inline_escape)
     esc = utils.escape(esc)
 
-    local exec_pat = "()"..inline_escape.."(%b"..brackets..")()"
+    local exec_pat = "()" .. inline_escape .. "(%b" .. brackets .. ")()"
 
-    local esc_pat = esc.."+([^\n]*\n?)"
-    local esc_pat1, esc_pat2 = "^"..esc_pat, "\n"..esc_pat
-    local  pieces, s = {"return function()\nlocal __R_size, __R_table, __tostring = 0, {}, __tostring", n = 1}, 1
+    local esc_pat = esc .. "+([^\n]*\n?)"
+    local esc_pat1, esc_pat2 = "^" .. esc_pat, "\n" .. esc_pat
+    local pieces, s = {"return function() local __R_size, __R_table, __tostring = 0, {}, __tostring; "}, 1
     while true do
-        local _, e, lua = strfind(chunk,esc_pat1, s)
+        local _, e, lua = strfind(chunk, esc_pat1, s)
         if not e then
             local ss
-            ss, e, lua = strfind(chunk,esc_pat2, s)
-            parseDollarParen(pieces, strsub(chunk,s, ss), exec_pat, newline)
+            ss, e, lua = strfind(chunk, esc_pat2, s)
+            parseDollarParen(pieces, strsub(chunk, s, ss), exec_pat, newline)
             if not e then break end
         end
-        if strsub(lua, -1, -1) == "\n" then lua = strsub(lua, 1, -2) end
-        append(pieces, "\n"..lua)
+        if strsub(lua, -1, -1) ~= "\n" then lua = lua .. "\n" end -- Ensure trailing newline
+        append(pieces, lua)
+        -- since `lua` ends with a newline, there is no danger of subsequent
+        -- statements being gobbled up by comments or being altered
         s = e + 1
     end
-    append(pieces, "\nreturn __R_table\nend")
+    append(pieces, "return __R_table; end")
 
     -- let's check for a special case where there is nothing to template, but it's
     -- just a single static string
     local short = false
-    if (#pieces == 3) and (pieces[2]:find(APPENDER, 1, true) == 1) then
-        pieces = { "return " .. pieces[2]:sub(#APPENDER+1,-1) }
+    if (#pieces == 3) and (strfind(pieces[2], APPENDER, 1, true) == 1) then
+        pieces = { "return " .. strsub(pieces[2], #APPENDER + 1, -1) }
         short = true
     end
     -- if short == true, the generated function will not return a table of strings,
     -- but a single string
-    return table.concat(pieces), short
+    return concat(pieces), short
 end
 
 local template = {}
@@ -108,19 +117,19 @@ local template = {}
 -- @tab[opt] env the environment
 -- @return `rendered template + nil + source_code`, or `nil + error + source_code`. The last
 -- return value (`source_code`) is only returned if the debug option is used.
-function template.substitute(str,env)
+function template.substitute(str, env)
     env = env or {}
     local t, err = template.compile(str, {
-        chunk_name = rawget(env,"_chunk_name"),
-        escape = rawget(env,"_escape"),
-        inline_escape = rawget(env,"_inline_escape"),
-        inline_brackets = rawget(env,"_brackets"),
+        chunk_name = rawget(env, "_chunk_name"),
+        escape = rawget(env, "_escape"),
+        inline_escape = rawget(env, "_inline_escape"),
+        inline_brackets = rawget(env, "_brackets"),
         newline = false,
-        debug = rawget(env,"_debug")
+        debug = rawget(env, "_debug")
     })
     if not t then return t, err end
 
-    return t:render(env, rawget(env,"_parent"), rawget(env,"_debug"))
+    return t:render(env, rawget(env, "_parent"), rawget(env, "_debug"))
 end
 
 --- executes the previously compiled template and renders it.
@@ -134,7 +143,7 @@ end
 -- @usage
 -- local ct, err = template.compile(my_template)
 -- local rendered , err = ct:render(my_env, parent)
-local render = function(self, env, parent, db)
+local function render(self, env, parent, db)
     env = env or {}
     if parent then  -- parent is a bit silly, but for backward compatibility retained
         setmetatable(env, {__index = parent})
@@ -146,13 +155,14 @@ local render = function(self, env, parent, db)
         if self.code and db then print(self.code) end
         return nil, out, self.code
     end
-    return table.concat(out), nil, self.code
+    return concat(out), nil, self.code
 end
 
 --- compiles the template.
 -- Returns an object that can repeatedly be rendered without parsing/compiling
--- the template again.
--- The options passed in the `opts` table support the following options:
+-- the template again. Preserves the line layout of the template so that line
+-- numbers in error messages should point to the correct lines in the source
+-- string. The options passed in the `opts` table support the following options:
 --
 --   * `chunk_name`: chunk name for loaded templates, used if there
 --     is an error in Lua code. Default is 'TMP'.
@@ -175,9 +185,9 @@ function template.compile(str, opts)
     local inline_escape = opts.inline_escape or '$'
     local inline_brackets = opts.inline_brackets or '()'
 
-    local code, short = parseHashLines(str,inline_escape,inline_brackets,escape,opts.newline)
+    local code, short = parseHashLines(str, inline_escape, inline_brackets, escape, opts.newline)
     local env = { __tostring = tostring }
-    local fn, err = utils.load(code, chunk_name,'t',env)
+    local fn, err = utils.load(code, chunk_name, 't', env)
     if not fn then return nil, err, code end
 
     if short then

tests/test-template.lua

@@ -262,7 +262,63 @@ asserteq(code, [[return "<ul>\
 <p>a paragraph</p>\
 <p>a paragraph</p>\
 </ul>\
-"]])
+";]])
+
+
+
+--------------------------------------------------
+-- Test template preserves line numbers, both when
+-- stripping and not stripping newlines
+local tmpl = [[
+# local function foo(x)
+  a$(x)b$(x + 1)c
+# return x + 2
+# end
+Hello
+there
+
+
+# foo(1)
+foo$(foo)bar
+# --
+]]
+local expected_num = select(2, string.gsub(tmpl, "\n", "\n"))
+
+-- Trailing newline, no newline stripping
+local t, err = template.compile(tmpl, { debug = true })
+local res, err, code = t:render(my_env)
+--print(res, err, code)
+
+assert(res, "rendering should not fail here")
+asserteq(select(2, string.gsub(code, "\n", "\n")), expected_num)
+
+-- Trailing newline, with newline stripping
+local t, err = template.compile(tmpl, { debug = true, newline = true })
+local res, err, code = t:render(my_env)
+--print(res, err, code)
+
+assert(res, "rendering should not fail here")
+asserteq(select(2, string.gsub(code, "\n", "\n")), expected_num)
+
+
+tmpl = string.sub(code, 1, -2) -- Remove the trailing newline
+-- num_expected remains unchanged because the template will append a trailing newline
+
+-- No trailing newline, no newline stripping
+local t, err = template.compile(tmpl, { debug = true })
+local res, err, code = t:render(my_env)
+--print(res, err, code)
+
+assert(res, "rendering should not fail here")
+asserteq(select(2, string.gsub(code, "\n", "\n")), expected_num)
+
+-- No trailing newline, with newline stripping
+local t, err = template.compile(tmpl, { debug = true, newline = true })
+local res, err, code = t:render(my_env)
+--print(res, err, code)
+
+assert(res, "rendering should not fail here")
+asserteq(select(2, string.gsub(code, "\n", "\n")), expected_num)
 
 
 print("template: success")