Module:Wikitext Parsing - Revision history

Dinoguy1000: the Lua # operator recalculates the length every time it's accessed (including on every loop in a for), so cache its value in cheaptrim since the string isn't modified in-place

2025-05-05T17:47:12Z

the Lua # operator recalculates the length every time it's accessed (including on every loop in a for), so cache its value in cheaptrim since the string isn't modified in-place

← Previous revision		Revision as of 17:47, 5 May 2025
Line 22:		Line 22:
	-- local out = string.gsub(str, "^%s(.-)%s$", "%1")		-- local out = string.gsub(str, "^%s(.-)%s$", "%1")
	local lowEnd		local lowEnd
	~~for~~ i = 1,#str do		local strlen = #str
			for i = 1,strlen do
	if not whitespace[string.sub(str, i, i)] then		if not whitespace[string.sub(str, i, i)] then
	lowEnd = i		lowEnd = i
Line 32:		Line 33:
	return ""		return ""
	end		end
	for i = ~~#str~~,1,-1 do		for i = strlen,1,-1 do
	if not whitespace[string.sub(str, i, i)] then		if not whitespace[string.sub(str, i, i)] then
	local out = string.sub(str, lowEnd, i)		local out = string.sub(str, lowEnd, i)

Pppery: Changed protection settings for "Module:Wikitext Parsing": High-risk template or module: This is, among other things, an indirect dependency of Module:Pagetype and hence Template:Short description, and hence used on 26% of all pages ([Edit=Require administrator access] (indefinite) [Move=Require administrator access] (indefinite))

2024-02-22T04:42:24Z

Changed protection settings for "Module:Wikitext Parsing": High-risk template or module: This is, among other things, an indirect dependency of Module:Pagetype and hence Template:Short description, and hence used on 26% of all pages ([Edit=Require administrator access] (indefinite) [Move=Require administrator access] (indefinite))

← Previous revision	Revision as of 04:42, 22 February 2024
(No difference)

Aidan9382: Fix weird behaviour with

 tags; simplify tag-end searching to avoid double-scanning

2024-01-31T21:54:57Z

Fix weird behaviour with <pre> tags; simplify tag-end searching to avoid double-scanning

← Previous revision		Revision as of 21:54, 31 January 2024
Line 56:		Line 56:
	If a tag has no end, it will consume all text instead of not processing		If a tag has no end, it will consume all text instead of not processing

	---- NOPROCESSING TAGS (nowiki, pre, syntaxhighlight, source) ----		---- NOPROCESSING TAGS (nowiki, pre, syntaxhighlight, source, etc.) ----
	(In most comments, <source> will not be mentioned. This is because it is the		(In most comments, <source> will not be mentioned. This is because it is the
	deprecated version of <syntaxhighlight>)		deprecated version of <syntaxhighlight>)
Line 64:		Line 64:
	follow the same set of rules, A closing tag can't have any sort of extra		follow the same set of rules, A closing tag can't have any sort of extra
	formatting period. While </div a/a> is valid, </nowiki a/a> isn't - only		formatting period. While </div a/a> is valid, </nowiki a/a> isn't - only
	newlines and spaces are allowed in closing tags ~~(except in <pre> tags, which~~		newlines and spaces/tabs are allowed in closing tags.
			Note that, even though <pre> tags cause a visual change when the ending tag has
	follow the rules of a regular html tag for formatting).
			extra formatting, it won't cause the no-processing effects. For some reason, the
			format must be strict for that to apply.

	Both the content inside the tag pair and the content inside each side of the		Both the content inside the tag pair and the content inside each side of the
Line 125:		Line 127:

	else		else
		⚫	local endingTagStart, endingTagEnd = string.find(text, "</"..allcases(tagName).."[ \t\n]*>", scanPosition)
	local endingTag
	if ~~tagName == "pre"~~ then --~~Looser restrictions~~ ~~for~~ ~~<pre>~~		if endingTagStart then --Regular tag formation
	endingTag = ~~--no \| so we just use~~ 2 ~~matches~~		local endingTag = string.sub(text, endingTagStart, endingTagEnd)
	string.~~match~~(text, ~~"</[Pp][Rr][Ee]>"~~, ~~scanPosition~~) or		local tagContent = string.sub(text, nextCloser+1, endingTagStart-1)
	string.match(text, "</[Pp][Rr][Ee][ \t\n/][^<]*>", scanPosition)
	else
⚫	~~endingTag~~ = string.~~match~~(text, "</"..allcases(tagName).."[ \t\n]*>", scanPosition)
	end

	if endingTag then --Regular tag formation
	local endingTagPosition = string.find(text, endingTag, nextCloser, true)
	local tagContent = string.sub(text, nextCloser+1, endingTagPosition-1)
	return {		return {
	Tag = tagName,		Tag = tagName,

MusikBot II: Protected "Module:Wikitext Parsing": High-risk template or module: 680413 transclusions (more info) ([Edit=Require template editor access] (indefinite) [Move=Require template editor access] (indefinite))

2023-04-24T18:00:07Z

Protected "Module:Wikitext Parsing": High-risk template or module: 680413 transclusions (more info) ([Edit=Require template editor access] (indefinite) [Move=Require template editor access] (indefinite))

← Previous revision	Revision as of 18:00, 24 April 2023
(No difference)

Aidan9382: regex correction for the start of TFC

2023-04-23T19:05:06Z

regex correction for the start of TFC

← Previous revision		Revision as of 19:05, 23 April 2023
Line 158:		Line 158:
	end		end
	local function TestForComment(text, scanPosition) --Like TestForNowikiTag but for <!-- -->		local function TestForComment(text, scanPosition) --Like TestForNowikiTag but for <!-- -->
	if string.match(text, "^<!--", scanPosition) then		if string.match(text, "^<!%-%-", scanPosition) then
	local commentEnd = string.find(text, "-->", scanPosition+4, true)		local commentEnd = string.find(text, "-->", scanPosition+4, true)
	if commentEnd then		if commentEnd then

Aidan9382: experiment with scanPosition

2023-04-23T18:10:01Z

experiment with scanPosition

← Previous revision		Revision as of 18:10, 23 April 2023
Line 105:		Line 105:
	local validtags = {nowiki=1, pre=1, syntaxhighlight=1, source=1, math=1}		local validtags = {nowiki=1, pre=1, syntaxhighlight=1, source=1, math=1}
	--This function expects the string to start with the tag		--This function expects the string to start with the tag
	local function TestForNowikiTag(text)		local function TestForNowikiTag(text, scanPosition)
	local tagName = (string.match(text, "^<([^\n />]+)") or ""):lower()		local tagName = (string.match(text, "^<([^\n />]+)", scanPosition) or ""):lower()
	if not validtags[tagName] then		if not validtags[tagName] then
	return nil		return nil
	end		end
	local nextOpener = string.find(text, "<", 2) or -1		local nextOpener = string.find(text, "<", scanPosition+1) or -1
	local nextCloser = string.find(text, ">", 2) or -1		local nextCloser = string.find(text, ">", scanPosition+1) or -1
	if nextCloser > -1 and (nextOpener == -1 or nextCloser < nextOpener) then		if nextCloser > -1 and (nextOpener == -1 or nextCloser < nextOpener) then
	local startingTag = string.sub(text, 1, nextCloser)		local startingTag = string.sub(text, scanPosition, nextCloser)
	--We have our starting tag (E.g. '<pre style="color:red">')		--We have our starting tag (E.g. '<pre style="color:red">')
	--Now find our ending...		--Now find our ending...
Line 128:		Line 128:
	if tagName == "pre" then --Looser restrictions for <pre>		if tagName == "pre" then --Looser restrictions for <pre>
	endingTag = --no \| so we just use 2 matches		endingTag = --no \| so we just use 2 matches
	string.match(text, "</[Pp][Rr][Ee]>") or		string.match(text, "</[Pp][Rr][Ee]>", scanPosition) or
	string.match(text, "</[Pp][Rr][Ee][ \t\n/][^<]*>")		string.match(text, "</[Pp][Rr][Ee][ \t\n/][^<]*>", scanPosition)
	else		else
	endingTag = string.match(text, "</"..allcases(tagName).."[ \t\n]*>")		endingTag = string.match(text, "</"..allcases(tagName).."[ \t\n]*>", scanPosition)
	end		end

Line 157:		Line 157:
	return nil		return nil
	end		end
	local function TestForComment(text) --Like TestForNowikiTag but for <!-- -->		local function TestForComment(text, scanPosition) --Like TestForNowikiTag but for <!-- -->
	if ~~startswith~~(text, "<!--") then		if string.match(text, "^<!--", scanPosition) then
	local commentEnd = string.find(text, "-->", 5, true)		local commentEnd = string.find(text, "-->", scanPosition+4, true)
	if commentEnd then		if commentEnd then
	return {		return {
	Start = "<!--", End = "-->",		Start = "<!--", End = "-->",
	Content = string.sub(text, 5, commentEnd-1),		Content = string.sub(text, scanPosition+4, commentEnd-1),
	Length = commentEnd+2		Length = commentEnd-scanPosition+3
	}		}
	else --Consumes all text if not given an ending		else --Consumes all text if not given an ending
	return {		return {
	Start = "<!--", End = "",		Start = "<!--", End = "",
	Content = string.sub(text, 5),		Content = string.sub(text, scanPosition+4),
	Length = #text		Length = #text-scanPosition+1
	}		}
	end		end
Line 186:		Line 186:
	local function PrepareText(text, keepComments)		local function PrepareText(text, keepComments)
	local newtext = {}		local newtext = {}
			local scanPosition = 1
	while ~~text ~= ""~~ do		while true do
	local NextCheck = string.find(text, "<[NnSsPpMm!]") --Advance to the next potential tag we care about		local NextCheck = string.find(text, "<[NnSsPpMm!]", scanPosition) --Advance to the next potential tag we care about
	if not NextCheck then --Done		if not NextCheck then --Done
	newtext[#newtext+1] = text		newtext[#newtext+1] = string.sub(text,scanPosition)
	break		break
	end		end
	newtext[#newtext+1] = string.sub(text,1,NextCheck-1)		newtext[#newtext+1] = string.sub(text,scanPosition,NextCheck-1)
	~~text~~ = ~~string.sub(text,~~ NextCheck)		scanPosition = NextCheck
	local Comment = TestForComment(text)		local Comment = TestForComment(text, scanPosition)
	if Comment then		if Comment then
	if keepComments then		if keepComments then
	newtext[#newtext+1] = Comment.Start .. mw.text.nowiki(Comment.Content) .. Comment.End		newtext[#newtext+1] = Comment.Start .. mw.text.nowiki(Comment.Content) .. Comment.End
	end		end
	~~text~~ = ~~string.sub(text,~~ Comment.Length~~+1)~~		scanPosition = scanPosition + Comment.Length
	else		else
	local Tag = TestForNowikiTag(text)		local Tag = TestForNowikiTag(text, scanPosition)
	if Tag then		if Tag then
	local newTagStart = "<" .. mw.text.nowiki(string.sub(Tag.Start,2,-2)) .. ">"		local newTagStart = "<" .. mw.text.nowiki(string.sub(Tag.Start,2,-2)) .. ">"
Line 209:		Line 210:
	local newContent = mw.text.nowiki(Tag.Content)		local newContent = mw.text.nowiki(Tag.Content)
	newtext[#newtext+1] = newTagStart .. newContent .. newTagEnd		newtext[#newtext+1] = newTagStart .. newContent .. newTagEnd
	~~text~~ = ~~string.sub(text,~~ Tag.Length~~+1)~~		scanPosition = scanPosition + Tag.Length
	else --Nothing special, move on...		else --Nothing special, move on...
	newtext[#newtext+1] = string.sub(text, 1, 1)		newtext[#newtext+1] = string.sub(text, scanPosition, scanPosition)
	~~text~~ = ~~string.sub(text,~~ 2)		scanPosition = scanPosition + 1
	end		end
	end		end
Line 512:		Line 513:
	mw.logObject(out)		mw.logObject(out)

	local s = [=[<!--		local s = [=[B<!--
	Hey!		Hey!
	-->A]=]		-->A]=]
	local out = p.~~TestForComments~~(s)		local out = p.TestForComment(s, 2)
	mw.logObject(out); mw.log(string.sub(s, 1, out.Length))		mw.logObject(out); mw.log(string.sub(s, 2, out.Length))

	local a = p.ParseTemplates([=[		local a = p.ParseTemplates([=[

Aidan9382: remove cheapinsert calls and just inline

2023-04-23T17:47:00Z

remove cheapinsert calls and just inline

← Previous revision		Revision as of 17:47, 23 April 2023
Line 40:		Line 40:
	end		end
	end		end
	end
	local function cheapinsert(a, b) --Faster than table.insert(a, b, no c)
	a[#a+1] = b
	end		end

Line 284:		Line 281:
	function Container:HandleArgInput(character, internalcall)		function Container:HandleArgInput(character, internalcall)
	if not internalcall then		if not internalcall then
	~~cheapinsert(~~self.Text, character)		self.Text[#self.Text+1] = character
	end		end
	if character == "=" then		if character == "=" then
	if self.Key then		if self.Key then
	~~cheapinsert(~~self.Value, character)		self.Value[#self.Value+1] = character
	else		else
	self.Key = cheaptrim(self.Value and table.concat(self.Value, "") or "")		self.Key = cheaptrim(self.Value and table.concat(self.Value, "") or "")
Line 302:		Line 299:
	self.Key = self.finalise(self.Key)		self.Key = self.finalise(self.Key)
	self.Args[self.Key] = cheaptrim(self.Value)		self.Args[self.Key] = cheaptrim(self.Value)
	~~cheapinsert(~~self.ArgOrder, self.Key)		self.ArgOrder[#self.ArgOrder+1] = self.Key
	else		else
	local Key = tostring(self.LastIndex)		local Key = tostring(self.LastIndex)
	self.Args[Key] = self.Value		self.Args[Key] = self.Value
	~~cheapinsert(~~self.ArgOrder, Key)		self.ArgOrder[#self.ArgOrder+1] = Key
	self.LastIndex = self.LastIndex + 1		self.LastIndex = self.LastIndex + 1
	end		end
Line 315:		Line 312:
	end		end
	function Container:AppendText(text, ftext)		function Container:AppendText(text, ftext)
	~~cheapinsert(~~self.Text, (ftext or text))		self.Text[#self.Text+1] = (ftext or text)
	if not self.Value then		if not self.Value then
	self.Value = {}		self.Value = {}
Line 321:		Line 318:
	self.BeyondStart = self.BeyondStart or (#table.concat(self.Text, "") > 2)		self.BeyondStart = self.BeyondStart or (#table.concat(self.Text, "") > 2)
	if self.BeyondStart then		if self.BeyondStart then
	~~cheapinsert(~~self.Value, text)		self.Value[#self.Value+1] = text
	end		end
	end		end
Line 354:		Line 351:
	scannerPosition = Position+2 --+2 to pass the [[ / ]]		scannerPosition = Position+2 --+2 to pass the [[ / ]]
	if Character == "[" then --Add a [[ to the pending wikilink queue		if Character == "[" then --Add a [[ to the pending wikilink queue
	~~cheapinsert(~~openWikilinks, Position)		openWikilinks[#openWikilinks+1] = Position
	else --Pair up the ]] to any available [[		else --Pair up the ]] to any available [[
	if #openWikilinks >= 1 then		if #openWikilinks >= 1 then
Line 377:		Line 374:
	scannerPosition = Start --Get to the {{ / }} set		scannerPosition = Start --Get to the {{ / }} set
	if Character == "{" then --Add the {{+ set to the queue		if Character == "{" then --Add the {{+ set to the queue
	~~cheapinsert(~~openBrackets, {Start=Start, End=End})		openBrackets[#openBrackets+1] = {Start=Start, End=End}

	else --Pair up the }} to any available {{, accounting for {{{ / }}}		else --Pair up the }} to any available {{, accounting for {{{ / }}}
Line 397:		Line 394:

	if boundlen(OpenSet) >= 2 then --Still has enough data left, leave it in		if boundlen(OpenSet) >= 2 then --Still has enough data left, leave it in
	~~cheapinsert(~~openBrackets, OpenSet)		openBrackets[#openBrackets+1] = OpenSet
	end		end
	end		end
Line 440:		Line 437:
	end		end
	if LatestObject and Container.Type == "Template" then --Only templates count as children		if LatestObject and Container.Type == "Template" then --Only templates count as children
	~~cheapinsert(~~LatestObject.Children, Container)		LatestObject.Children[#LatestObject.Children+1] = Container
	end		end
	~~cheapinsert(~~activeObjects, Container)		activeObjects[#activeObjects+1] = Container
	elseif LatestObject then		elseif LatestObject then
	LatestObject:AppendText(Character)		LatestObject:AppendText(Character)
Line 453:		Line 450:
	if LatestObject.Type == "Template" then		if LatestObject.Type == "Template" then
	LatestObject:Clean(true)		LatestObject:Clean(true)
	~~cheapinsert(~~finalObjects, LatestObject)		finalObjects[#finalObjects+1] = LatestObject
	else		else
	LatestObject:Clean(false)		LatestObject:Clean(false)
Line 476:		Line 473:
	local SortableReference = {}		local SortableReference = {}
	for _,Object in next,finalObjects do		for _,Object in next,finalObjects do
	~~cheapinsert(~~SortableReference, Object.Start)		SortableReference[#SortableReference+1] = Object.Start
	end		end
	table.sort(SortableReference)		table.sort(SortableReference)
Line 487:		Line 484:
	Object.End = nil		Object.End = nil
	Object.Type = nil		Object.Type = nil
	~~cheapinsert(~~FixedOrder, Object)		FixedOrder[#FixedOrder+1] = Object
	break		break
	end		end

Aidan9382: woops, im just dumb

2023-04-23T17:44:23Z

woops, im just dumb

← Previous revision		Revision as of 17:44, 23 April 2023
Line 192:		Line 192:
	local NextCheck = string.find(text, "<[NnSsPpMm!]") --Advance to the next potential tag we care about		local NextCheck = string.find(text, "<[NnSsPpMm!]") --Advance to the next potential tag we care about
	if not NextCheck then --Done		if not NextCheck then --Done
	newtext[#newtext+1] = ~~newtext~~		newtext[#newtext+1] = text
	break		break
	end		end

Aidan9382: Re-test the supposedly (2+)x better option

2023-04-23T17:42:32Z

Re-test the supposedly (2+)x better option

← Previous revision		Revision as of 17:42, 23 April 2023
Line 188:		Line 188:
	--]]		--]]
	local function PrepareText(text, keepComments)		local function PrepareText(text, keepComments)
	local newtext = ""		local newtext = {}
	while text ~= "" do		while text ~= "" do
	local NextCheck = string.find(text, "<[NnSsPpMm!]") --Advance to the next potential tag we care about		local NextCheck = string.find(text, "<[NnSsPpMm!]") --Advance to the next potential tag we care about
	if not NextCheck then --Done		if not NextCheck then --Done
	newtext = newtext ~~.. text~~		newtext[#newtext+1] = newtext
	break		break
	end		end
	newtext = ~~newtext ..~~ string.sub(text,1,NextCheck-1)		newtext[#newtext+1] = string.sub(text,1,NextCheck-1)
	text = string.sub(text, NextCheck)		text = string.sub(text, NextCheck)
	local Comment = TestForComment(text)		local Comment = TestForComment(text)
	if Comment then		if Comment then
	if keepComments then		if keepComments then
	newtext = ~~newtext ..~~ Comment.Start .. mw.text.nowiki(Comment.Content) .. Comment.End		newtext[#newtext+1] = Comment.Start .. mw.text.nowiki(Comment.Content) .. Comment.End
	end		end
	text = string.sub(text, Comment.Length+1)		text = string.sub(text, Comment.Length+1)
Line 211:		Line 211:
	"</" .. mw.text.nowiki(string.sub(Tag.End,3,-2)) .. ">"		"</" .. mw.text.nowiki(string.sub(Tag.End,3,-2)) .. ">"
	local newContent = mw.text.nowiki(Tag.Content)		local newContent = mw.text.nowiki(Tag.Content)
	newtext = ~~newtext ..~~ newTagStart .. newContent .. newTagEnd		newtext[#newtext+1] = newTagStart .. newContent .. newTagEnd
	text = string.sub(text, Tag.Length+1)		text = string.sub(text, Tag.Length+1)
	else --Nothing special, move on...		else --Nothing special, move on...
	newtext = ~~newtext ..~~ string.sub(text, 1, 1)		newtext[#newtext+1] = string.sub(text, 1, 1)
	text = string.sub(text, 2)		text = string.sub(text, 2)
	end		end
	end		end
	end		end
	return newtext		return table.concat(newtext, "")
	end		end

Aidan9382: I just need to quickly confirm if this change is really saving as much as it claims it did

2023-04-23T17:41:49Z

I just need to quickly confirm if this change is really saving as much as it claims it did

← Previous revision		Revision as of 17:41, 23 April 2023
Line 188:		Line 188:
	--]]		--]]
	local function PrepareText(text, keepComments)		local function PrepareText(text, keepComments)
	local newtext = {}		local newtext = ""
	while text ~= "" do		while text ~= "" do
	local NextCheck = string.find(text, "<[NnSsPpMm!]") --Advance to the next potential tag we care about		local NextCheck = string.find(text, "<[NnSsPpMm!]") --Advance to the next potential tag we care about
	if not NextCheck then --Done		if not NextCheck then --Done
	newtext~~[#newtext+1]~~ = newtext		newtext = newtext .. text
	break		break
	end		end
	newtext~~[#newtext+1]~~ = string.sub(text,1,NextCheck-1)		newtext = newtext .. string.sub(text,1,NextCheck-1)
	text = string.sub(text, NextCheck)		text = string.sub(text, NextCheck)
	local Comment = TestForComment(text)		local Comment = TestForComment(text)
	if Comment then		if Comment then
	if keepComments then		if keepComments then
	newtext~~[#newtext+1]~~ = Comment.Start .. mw.text.nowiki(Comment.Content) .. Comment.End		newtext = newtext .. Comment.Start .. mw.text.nowiki(Comment.Content) .. Comment.End
	end		end
	text = string.sub(text, Comment.Length+1)		text = string.sub(text, Comment.Length+1)
Line 211:		Line 211:
	"</" .. mw.text.nowiki(string.sub(Tag.End,3,-2)) .. ">"		"</" .. mw.text.nowiki(string.sub(Tag.End,3,-2)) .. ">"
	local newContent = mw.text.nowiki(Tag.Content)		local newContent = mw.text.nowiki(Tag.Content)
	newtext~~[#newtext+1]~~ = newTagStart .. newContent .. newTagEnd		newtext = newtext .. newTagStart .. newContent .. newTagEnd
	text = string.sub(text, Tag.Length+1)		text = string.sub(text, Tag.Length+1)
	else --Nothing special, move on...		else --Nothing special, move on...
	newtext~~[#newtext+1]~~ = string.sub(text, 1, 1)		newtext = newtext .. string.sub(text, 1, 1)
	text = string.sub(text, 2)		text = string.sub(text, 2)
	end		end
	end		end
	end		end
	return ~~table.concat(~~newtext~~, "")~~		return newtext
	end		end