Difference between revisions of "Module:Language/scripts"

From blackwiki
Jump to navigation Jump to search
blackwiki>Erutuon
(testing memory usage if script codes are replaced with numbers)
m (34 revisions imported)
 
(11 intermediate revisions by 3 users not shown)
Line 1: Line 1:
 
local p = {}
 
local p = {}
 
 
local gsub = mw.ustring.gsub
 
local gsub = mw.ustring.gsub
 
local length = mw.ustring.len
 
local length = mw.ustring.len
 
local floor = math.floor
 
local floor = math.floor
local table_size = require("Module:TableTools").size
 
 
local UTF8Char = "[%z\1-\127\194-\244][\128-\191]*"
 
local UTF8Char = "[%z\1-\127\194-\244][\128-\191]*"
 +
 +
local codepoint_data = mw.loadData("Module:language/scripts/codepoints")
  
 
local data = require("Module:Language/scripts/data")
 
local data = require("Module:Language/scripts/data")
Line 45: Line 45:
 
return p.isLatn(text)
 
return p.isLatn(text)
 
end
 
end
 +
 +
local ignore_script = require("Module:TableTools").listToSet{
 +
"Zinh", "Zyyy", "Zzzz"
 +
}
  
 
local function map(func, t)
 
local function map(func, t)
Line 50: Line 54:
 
if t[1] then
 
if t[1] then
 
for i, v in ipairs(t) do
 
for i, v in ipairs(t) do
array[i] = func(i, v, t)
+
array[i] = func(v, i, t)
 
end
 
end
 
else
 
else
Line 56: Line 60:
 
for k, v in pairs(t) do
 
for k, v in pairs(t) do
 
i = i + 1
 
i = i + 1
array[i] = func(k, v, t)
+
array[i] = func(v, k, t)
 
end
 
end
 
end
 
end
 
return array
 
return array
 +
end
 +
 +
local function filter(t, func)
 +
local new_t = {}
 +
 +
if t[1] then
 +
local new_t_i = 0
 +
for i, v in ipairs(t) do
 +
if func(v, i, t) then
 +
new_t_i = new_t_i + 1
 +
new_t[new_t_i] = v
 +
end
 +
end
 +
else
 +
for k, v in pairs(t) do
 +
if func(v, k, t) then
 +
new_t[k] = v
 +
end
 +
end
 +
end
 +
 +
return new_t
 
end
 
end
  
Line 75: Line 101:
 
 
 
-- Initialize numbers.
 
-- Initialize numbers.
local bottom, middle = 1, 0
+
local bottom, i, top = 1, 0, ranges.length
-- Can't use # because table is loaded by mw.loadData.
 
local top = table_size(ranges)
 
  
 
if top == 0 then
 
if top == 0 then
Line 85: Line 109:
 
-- Do search.
 
-- Do search.
 
while bottom <= top do
 
while bottom <= top do
-- Calculate middle.
+
-- Calculate current index.
middle = floor((bottom + top) / 2)
+
i = floor((bottom + top) / 2)
  
-- Get compare value.
+
-- Get range array; for instance, { 0x41, 0x7A, "Latn"}.
local range = ranges[middle]
+
local range = ranges[i]
  
 
if value < range[1] then
 
if value < range[1] then
top = middle - 1
+
top = i - 1
  
-- Return matching index. Assumes there are no duplicates.
+
-- Return matching range array so that it can be placed in cache.
 
elseif value <= range[2] then
 
elseif value <= range[2] then
 
return range
 
return range
  
-- Keep searching.
 
 
else
 
else
bottom = middle + 1
+
bottom = i + 1
 
end
 
end
 
end
 
end
 +
 
return nil
 
return nil
 
end
 
end
Line 141: Line 165:
 
]=]
 
]=]
 
function p.codepointToScript(codepoint)
 
function p.codepointToScript(codepoint)
local lookup = mw.loadData("Module:language/scripts/codepoints")
+
local lookup = codepoint_data
 
local t = type(codepoint)
 
local t = type(codepoint)
 
if t ~= "number" then
 
if t ~= "number" then
Line 149: Line 173:
 
local individualMatch = lookup.individual[codepoint]
 
local individualMatch = lookup.individual[codepoint]
 
if individualMatch then
 
if individualMatch then
return lookup.numberToScriptCode[individualMatch]
+
return individualMatch
 
else
 
else
 
local script = lookUpInOrder(codepoint, rangesCache)
 
local script = lookUpInOrder(codepoint, rangesCache)
 
if script then
 
if script then
return lookup.numberToScriptCode[script]
+
return script
 
end
 
end
  
Line 160: Line 184:
 
table.insert(rangesCache, range)
 
table.insert(rangesCache, range)
 
table.sort(rangesCache, sortRange)
 
table.sort(rangesCache, sortRange)
return lookup.numberToScriptCode[range[3]]
+
return range[3]
 
end
 
end
 
end
 
end
Line 167: Line 191:
 
end
 
end
  
local function charToScript(char)
+
function p.charToScript(char)
 
return p.codepointToScript(mw.ustring.codepoint(char))
 
return p.codepointToScript(mw.ustring.codepoint(char))
 
end
 
end
Line 188: Line 212:
 
 
 
return scriptCounts
 
return scriptCounts
 +
end
 +
 +
function p.getScript(text)
 +
local scripts = {}
 +
local i = 0
 +
for code in pairs(p.countScripts(text)) do
 +
i = i + 1
 +
scripts[i] = code
 +
end
 +
 +
scripts = filter(scripts,
 +
function (scCode)
 +
return not ignore_script[scCode]
 +
end)
 +
 +
if not scripts[2] then
 +
return scripts[1]
 +
end
 
end
 
end
  
 
function p.showScripts(frame)
 
function p.showScripts(frame)
 
return table.concat(
 
return table.concat(
map(function(i, arg)
+
map(function(arg)
 
return "* " .. arg .. ": " .. table.concat(
 
return "* " .. arg .. ": " .. table.concat(
map(function(k, v)
+
map(function(count, script)
return k .. " (" .. v .. ")"
+
return script .. " (" .. count .. ")"
 
end,
 
end,
 
p.countScripts(arg)),
 
p.countScripts(arg)),

Latest revision as of 05:55, 27 September 2020

Testcases

All tests passed.

testcases for countScripts
Text Expected Actual
☑Y Ста́нция Восто́к Cyrl (13), Zinh (2), Zyyy (7) Cyrl (13), Zinh (2), Zyyy (7)
☑Y Σωκράτης Grek (8) Grek (8)
☑Y 中华人民共和国 Hani (7) Hani (7)
☑Y অবনী বাড়ি আছো Ôboni Baŗi Achho Beng (12), Latn (14), Zyyy (9) Beng (12), Latn (14), Zyyy (9)
testcases for getScript
Text Expected Actual
☑Y Ста́нция Восто́к Cyrl Cyrl
☑Y Ἑλλήσποντος Grek Grek


Invokable function

{{#invoke:Language/scripts|showScripts|lá:yelhp|Quw̓utsun̓|Hul̓q̓umín̓um̓ / hən̓q̓əmin̓əm̓|xʷməθkʷəy̓əm|hən̓q̓əmin̓əm|Hul’q’umi’num’/Halq'eméyle/hən̓q̓əmin̓əm|sc̓əwaθən məsteyəxʷ|c̓əsnaʔəm}}
  • lá:yelhp: Latn (7), Zyyy (1)
  • Quw̓utsun̓: Latn (8), Zinh (2)
  • Hul̓q̓umín̓um̓ / hən̓q̓əmin̓əm̓: Latn (20), Zinh (8), Zyyy (3)
  • xʷməθkʷəy̓əm: Latn (10), Grek (1), Zinh (1)
  • hən̓q̓əmin̓əm: Latn (10), Zinh (3)
  • Hul’q’umi’num’/Halq'eméyle/hən̓q̓əmin̓əm: Latn (30), Zyyy (7), Zinh (3)
  • sc̓əwaθən məsteyəxʷ: Latn (16), Zinh (1), Zyyy (1), Grek (1)
  • c̓əsnaʔəm: Latn (8), Zinh (1)

local p = {}
local gsub = mw.ustring.gsub
local length = mw.ustring.len
local floor = math.floor
local UTF8Char = "[%z\1-\127\194-\244][\128-\191]*"

local codepoint_data = mw.loadData("Module:language/scripts/codepoints")

local data = require("Module:Language/scripts/data")

function p.print(frame)
	local scriptCode = frame.args[1]
	local scriptData = scriptCode and data[scriptCode] or "Please supply a valid script code."
	local characters = scriptData and scriptData.characters or "No characters found for " .. scriptCode .. "."
	return characters
end

local script = {}

-- Based on the Script:countCharacters() function of Module:scripts on Wiktionary
local function countCharacters(text, scriptCode)
	if not data[scriptCode]["characters"] then
		return 0
	else
		local _, count = gsub(text, "[" .. data[scriptCode]["characters"] .. "]", "")
		return count
	end
end

function p.isLatn(text)
	if type(tostring(text)) == "string" then
		local count = countCharacters(text, "Latn")
		if count < (length(text) / 4) then -- Only 25% of characters in string are Latin
			return false
		else
			return true
		end
	else
		return nil
	end
end

function p.Latin(frame)
	local text = frame.args[1]
	return p.isLatn(text)
end

local ignore_script = require("Module:TableTools").listToSet{
	"Zinh", "Zyyy", "Zzzz"
}

local function map(func, t)
	local array = {}
	if t[1] then
		for i, v in ipairs(t) do
			array[i] = func(v, i, t)
		end
	else
		local i = 0
		for k, v in pairs(t) do
			i = i + 1
			array[i] = func(v, k, t)
		end
	end
	return array
end

local function filter(t, func)
	local new_t = {}
	
	if t[1] then
		local new_t_i = 0
		for i, v in ipairs(t) do
			if func(v, i, t) then
				new_t_i = new_t_i + 1
				new_t[new_t_i] = v
			end
		end
	else
		for k, v in pairs(t) do
			if func(v, k, t) then
				new_t[k] = v
			end
		end
	end
	
	return new_t
end

local function sortRange(range1, range2)
	return range1[1] < range2[1]
end

--[[
	Binary search: efficient for long lists of codepoint ranges.
]]
local function binarySearch(ranges, value)
	if not ranges then
		return nil
	end
	
	--	Initialize numbers.
	local bottom, i, top = 1, 0, ranges.length

	if top == 0 then
		return nil
	end

	-- Do search.
	while bottom <= top do
		-- Calculate current index.
		i = floor((bottom + top) / 2)

		-- Get range array; for instance, { 0x41, 0x7A, "Latn"}.
		local range = ranges[i]

		if value < range[1] then
			top = i - 1

		-- Return matching range array so that it can be placed in cache.
		elseif value <= range[2] then
			return range

		else
			bottom = i + 1
		end
	end
	
	return nil
end

--[[
-- For debugging
local function toHex(number)
	return ("0x%X"):format(number)
end

local function logRange(range, number)
	return mw.log(toHex(range[1]), toHex(number) .. " (" .. mw.ustring.char(number) .. ")", toHex(range[2]), range[3])
end
--]]

local function lookUpInOrder(number, ranges)
	for i, range in ipairs(ranges) do
		if number < range[1] then
			return nil
		elseif number <= range[2] then
			return range[3]
		end
	end
end

-- Save previously used codepoint ranges in case another character is in the
-- same range.
local rangesCache = {}

--[=[
	Takes a codepoint and returns the script code that is appropriate for it,
	based on the data module [[Module:Language/scripts/codepoints]].
	
	The data module uses the official Unicode script codes.

	Returns a script code from the codepoint-to-script map, or one of the ranges
	in the array of ranges, else returns Zzzz.
]=]
function p.codepointToScript(codepoint)
	local lookup = codepoint_data
	local t = type(codepoint)
	if t ~= "number" then
		error("Argument to codepointToScript should be a number, but its type is " .. t .. ".")
	end

	local individualMatch = lookup.individual[codepoint]
	if individualMatch then
		return individualMatch
	else
		local script = lookUpInOrder(codepoint, rangesCache)
		if script then
			return script
		end

		local range = binarySearch(lookup.ranges, codepoint)
		if range then
			table.insert(rangesCache, range)
			table.sort(rangesCache, sortRange)
			return range[3]
		end
	end

	return "Zzzz"
end

function p.charToScript(char)
	return p.codepointToScript(mw.ustring.codepoint(char))
end

function p.countScripts(text)
	if type(text) ~= "string" then
		error("countScripts requires a string")
	end
	local scriptCounts = {}
	local codepointToScript = p.codepointToScript
	for codepoint in mw.ustring.gcodepoint(text) do
		local script = codepointToScript(codepoint)
		if script then
			if not scriptCounts[script] then
				scriptCounts[script] = 0
			end
			scriptCounts[script] = scriptCounts[script] + 1
		end
	end
	
	return scriptCounts
end

function p.getScript(text)
	local scripts = {}
	local i = 0
	for code in pairs(p.countScripts(text)) do
		i = i + 1
		scripts[i] = code
	end
	
	scripts = filter(scripts,
		function (scCode)
			return not ignore_script[scCode]
		end)
	
	if not scripts[2] then
		return scripts[1]
	end
end

function p.showScripts(frame)
	return table.concat(
		map(function(arg)
				return "* " .. arg .. ": " .. table.concat(
					map(function(count, script)
							return script .. " (" .. count .. ")"
						end,
						p.countScripts(arg)),
					", ")
			end,
			frame.args),
		"\n")
end

return p