Module:Unicode data/category/make - Revision history

Erutuon: be clearer about input

2020-03-11T19:22:33Z

be clearer about input

← Previous revision		Revision as of 19:22, 11 March 2020
Line 5:		Line 5:
	end		end

	function p.make_singles_and_ranges()		function p.make_singles_and_ranges(derived_general_category_page, property_value_aliases_page)
	local text = read_page ~~"User:Erutuon/Unicode/DerivedGeneralCategory.txt"~~		local text = read_page(derived_general_category_page)

	local singles, ranges = {}, {}		local singles, ranges = {}, {}
Line 25:		Line 25:
	end		end

	local property_value_aliases = read_page ~~"User:Erutuon/Unicode/PropertyValueAliases.txt"~~		local property_value_aliases = read_page(property_value_aliases_page)
	local general_category_aliases = property_value_aliases:match "# General_Category[^\n]+%s(.-)%s%f[^\n]#"		local general_category_aliases = property_value_aliases:match "# General_Category[^\n]+%s(.-)%s%f[^\n]#"
	local long_names = {}		local long_names = {}
Line 82:		Line 82:
	function p.main(frame)		function p.main(frame)
	local data = {}		local data = {}
	data.singles, data.ranges, data.long_names = p.make_singles_and_ranges()		data.singles, data.ranges, data.long_names = p.make_singles_and_ranges(
			"User:Erutuon/Unicode/DerivedGeneralCategory.txt",
			"User:Erutuon/Unicode/PropertyValueAliases.txt")
	return frame:extensionTag{		return frame:extensionTag{
	name = "syntaxhighlight",		name = "syntaxhighlight",

Erutuon: pattern missed the line 100000..10FFFD; Co # [65534] ..

2019-09-17T00:01:56Z

pattern missed the line 100000..10FFFD; Co # [65534] <private-use-100000>..<private-use-10FFFD>

← Previous revision		Revision as of 00:01, 17 September 2019
Line 9:		Line 9:

	local singles, ranges = {}, {}		local singles, ranges = {}, {}
	for code_point1, code_point2, general_category in text:gmatch "%f[^\n%z](%x+)%.?%.?(%x*)%s+;%s+(%u%l)" do		for code_point1, code_point2, general_category in text:gmatch "%f[^\n%z](%x+)%.?%.?(%x)%s;%s*(%u%l)" do
	if general_category ~= "Cn" then		if general_category ~= "Cn" then
	code_point1 = tonumber(code_point1, 16)		code_point1 = tonumber(code_point1, 16)

Erutuon: gah

2019-09-16T19:41:33Z

gah

← Previous revision		Revision as of 19:41, 16 September 2019
Line 10:		Line 10:
	local singles, ranges = {}, {}		local singles, ranges = {}, {}
	for code_point1, code_point2, general_category in text:gmatch "%f[^\n%z](%x+)%.?%.?(%x*)%s+;%s+(%u%l)" do		for code_point1, code_point2, general_category in text:gmatch "%f[^\n%z](%x+)%.?%.?(%x*)%s+;%s+(%u%l)" do
	if ~~category~~ ~= "Cn" then		if general_category ~= "Cn" then
	code_point1 = tonumber(code_point1, 16)		code_point1 = tonumber(code_point1, 16)

Erutuon: align long names

2019-09-16T19:16:54Z

align long names

← Previous revision		Revision as of 19:16, 16 September 2019
Line 70:		Line 70:
	]]		]]
	for abbr, long_name in require "Module:TableTools".sortedPairs(data.long_names) do		for abbr, long_name in require "Module:TableTools".sortedPairs(data.long_names) do
	writef('\t\t%s = "%s",\n', abbr, long_name)		writef('\t\t%-2s = "%s",\n', abbr, long_name)
	end		end

Erutuon: better to use PropertyValueAliases.txt

2019-09-16T19:15:22Z

better to use PropertyValueAliases.txt

← Previous revision		Revision as of 19:15, 16 September 2019
Line 1:		Line 1:
	local p = {}		local p = {}

			local function read_page(title)
			return mw.title.new(title):getContent()
			end

	function p.make_singles_and_ranges()		function p.make_singles_and_ranges()
	local text = ~~mw.title.new(~~"User:Erutuon/Unicode/DerivedGeneralCategory.txt"~~):getContent()~~		local text = read_page "User:Erutuon/Unicode/DerivedGeneralCategory.txt"

	local singles, ranges = {}, {}		local singles, ranges = {}, {}
	for code_point1, code_point2, general_category in text:gmatch("%f[^\n%z](%x+)%.?%.?(%x*)%s+;%s+(%u%l)") do		for code_point1, code_point2, general_category in text:gmatch "%f[^\n%z](%x+)%.?%.?(%x*)%s+;%s+(%u%l)" do
	if category ~= "Cn" then		if category ~= "Cn" then
	code_point1 = tonumber(code_point1, 16)		code_point1 = tonumber(code_point1, 16)
Line 21:		Line 25:
	end		end

			local property_value_aliases = read_page "User:Erutuon/Unicode/PropertyValueAliases.txt"
			local general_category_aliases = property_value_aliases:match "# General_Category[^\n]+%s(.-)%s%f[^\n]#"
	local long_names = {}		local long_names = {}
	for ~~long_name~~, ~~code~~ in ~~text~~:gmatch "~~# General_Category=~~([%~~a_]~~+).-%~~f[^\n]#.-~~; (%~~u%l~~)" do		for abbr, long_name in general_category_aliases:gmatch "gc%s;%s(%S+)%s;%s([%a_]+)" do
	long_names[~~code~~] = long_name		long_names[abbr] = long_name
	end		end

Line 63:		Line 69:
	long_names = {		long_names = {
	]]		]]
	for ~~code~~, long_name in require "Module:TableTools".sortedPairs(data.long_names) do		for abbr, long_name in require "Module:TableTools".sortedPairs(data.long_names) do
	writef('\t\t%s = "%s",\n', ~~code~~, long_name)		writef('\t\t%s = "%s",\n', abbr, long_name)
	end		end

Erutuon: add long names

2019-09-16T19:08:03Z

add long names

← Previous revision		Revision as of 19:08, 16 September 2019
Line 20:		Line 20:
	end		end
	end		end

⚫	return singles, ranges
			local long_names = {}
			for long_name, code in text:gmatch "# General_Category=([%a_]+).-%f[^\n]#.-; (%u%l)" do
			long_names[code] = long_name
			end

		⚫	return singles, ranges, long_names
	end		end

Line 51:		Line 57:
	for _, range in ipairs(data.ranges) do		for _, range in ipairs(data.ranges) do
	writef('\t\t{ 0x%06X, 0x%06X, "%s" },\n', unpack(range))		writef('\t\t{ 0x%06X, 0x%06X, "%s" },\n', unpack(range))
			end

			writef [[
			},
			long_names = {
			]]
			for code, long_name in require "Module:TableTools".sortedPairs(data.long_names) do
			writef('\t\t%s = "%s",\n', code, long_name)
	end		end

Line 62:		Line 76:
	function p.main(frame)		function p.main(frame)
	local data = {}		local data = {}
	data.singles, data.ranges = p.make_singles_and_ranges()		data.singles, data.ranges, data.long_names = p.make_singles_and_ranges()
	return frame:extensionTag{		return frame:extensionTag{
	name = "syntaxhighlight",		name = "syntaxhighlight",

Erutuon: fixed

2019-09-16T19:00:32Z

fixed

← Previous revision		Revision as of 19:00, 16 September 2019
Line 2:		Line 2:

	function p.make_singles_and_ranges()		function p.make_singles_and_ranges()
	local text = mw.title.new('DerivedGeneralCategory.txt'):getContent()		local text = mw.title.new("User:Erutuon/Unicode/DerivedGeneralCategory.txt"):getContent()

	local singles, ranges = {}, {}		local singles, ranges = {}, {}
	for code_point1, code_point2, general_category in text:gmatch("%f[^\n%z](%x+)%.?%.?(%x+)%s+;%s+(%u%l)") do		for code_point1, code_point2, general_category in text:gmatch("%f[^\n%z](%x+)%.?%.?(%x*)%s+;%s+(%u%l)") do
	if category ~= 'Cn' then		if category ~= "Cn" then
	code_point1 = tonumber(code_point1, 16)		code_point1 = tonumber(code_point1, 16)

Line 12:		Line 12:
	if code_point2 ~= "" then		if code_point2 ~= "" then
	code_point2 = tonumber(code_point2, 16)		code_point2 = tonumber(code_point2, 16)
	table.insert(ranges, { code_point1, code_point2, ~~category~~ })		table.insert(ranges, { code_point1, code_point2, general_category })

	-- XXXX ; gc		-- XXXX ; gc
	else		else
	singles[code_point1] = ~~category~~		singles[code_point1] = general_category
	end		end
	end		end
Line 35:		Line 35:

	-- Check that maximum "singles" codepoint is less than 0x100000?		-- Check that maximum "singles" codepoint is less than 0x100000?
	for codepoint, category in require ~~'t'~~.~~spairs~~(data.singles) do		for codepoint, category in require "Module:TableTools".sortedPairs(data.singles) do
	writef('\t\t[0x%05X] = "%s",\n', codepoint, category)		writef('\t\t[0x%05X] = "%s",\n', codepoint, category)
	end		end
Line 50:		Line 50:
	table.sort(data.ranges, compare_ranges)		table.sort(data.ranges, compare_ranges)
	for _, range in ipairs(data.ranges) do		for _, range in ipairs(data.ranges) do
	writef('\t\t{ 0x%06X, 0x%06X, "%s" },\n', ~~table.~~unpack(range))		writef('\t\t{ 0x%06X, 0x%06X, "%s" },\n', unpack(range))
	end		end

Line 62:		Line 62:
	function p.main(frame)		function p.main(frame)
	local data = {}		local data = {}
	data.singles, data.ranges = p.make_singles_and_ranges		data.singles, data.ranges = p.make_singles_and_ranges()
	return frame:extensionTag{		return frame:extensionTag{
	name = "syntaxhighlight",		name = "syntaxhighlight",

Erutuon: adaptation of overcomplicated script in Module:Unicode data/category/doc

2019-09-16T18:53:38Z

adaptation of overcomplicated script in Module:Unicode data/category/doc

New page

local p = {}

function p.make_singles_and_ranges()
local text = mw.title.new('DerivedGeneralCategory.txt'):getContent()

local singles, ranges = {}, {}
for code_point1, code_point2, general_category in text:gmatch("%f[^\n%z](%x+)%.?%.?(%x+)%s+;%s+(%u%l)") do
if category ~= 'Cn' then
code_point1 = tonumber(code_point1, 16)

-- XXXX..XXXX ; gc
if code_point2 ~= "" then
code_point2 = tonumber(code_point2, 16)
table.insert(ranges, { code_point1, code_point2, category })

-- XXXX ; gc
else
singles[code_point1] = category
end
end
end
return singles, ranges
end

function p.print_data(data)
local output = require "Module:array"()
local function writef(...)
output:insert(string.format(...))
end

writef [[
return {
singles = {
]]

-- Check that maximum "singles" codepoint is less than 0x100000?
for codepoint, category in require 't'.spairs(data.singles) do
writef('\t\t[0x%05X] = "%s",\n', codepoint, category)
end

writef [[
},
ranges = {
]]

local function compare_ranges(range1, range2)
return range1[1] < range2[1]
end

table.sort(data.ranges, compare_ranges)
for _, range in ipairs(data.ranges) do
writef('\t\t{ 0x%06X, 0x%06X, "%s" },\n', table.unpack(range))
end

writef [[
},
}]]

return output:concat()
end

function p.main(frame)
local data = {}
data.singles, data.ranges = p.make_singles_and_ranges
return frame:extensionTag{
name = "syntaxhighlight",
content = p.print_data(data),
args = { lang = "lua" }
}
end

return p