Difference between revisions of "Module:Language/scripts"

From blackwiki
Jump to navigation Jump to search
blackwiki>Erutuon
(now can at least recognize codepoints, ranges, and script names)
blackwiki>Erutuon
m (hex to number and back)
Line 225: Line 225:
 
local function pre(text)
 
local function pre(text)
 
return '<pre>' .. text .. '</pre>'
 
return '<pre>' .. text .. '</pre>'
 +
end
 +
 +
local function toHex(number)
 +
return ("0x%X"):format(number)
 +
end
 +
 +
local function fromHex(str)
 +
return tonumber(str, 16)
 
end
 
end
  
Line 236: Line 244:
 
local lower, higher, script = line:match("(%x+)%.%.(%x+) +; +(%a+)")
 
local lower, higher, script = line:match("(%x+)%.%.(%x+) +; +(%a+)")
 
if lower then
 
if lower then
ranges[i] = { lower, higher, script }
+
ranges[i] = { fromHex(lower), fromHex(higher), script }
 
i = i + 1
 
i = i + 1
 
else
 
else
 
local codepoint, script = line:match("(%x+) +; +(%a+)")
 
local codepoint, script = line:match("(%x+) +; +(%a+)")
 
if codepoint then
 
if codepoint then
individual[tonumber(codepoint, 16)] = script
+
individual[fromHex(codepoint)] = script
 
end
 
end
 
end
 
end
Line 254: Line 262:
 
map(
 
map(
 
function(range)
 
function(range)
return range[1] .. "–" .. range[2] .. ": " .. range[3]
+
return toHex(range[1]) .. "–" .. toHex(range[2]) .. ": " .. range[3]
 
end,
 
end,
 
ranges),
 
ranges),
Line 261: Line 269:
 
sparseMap(
 
sparseMap(
 
function(script, codepoint)
 
function(script, codepoint)
return ("0x%X: "):format(codepoint) .. script
+
return toHex(codepoint) .. ": " .. script
 
end,
 
end,
 
individual),
 
individual),

Revision as of 03:05, 14 November 2017

Testcases

Lua error in package.lua at line 80: module 'Module:Utility' not found.

Invokable function

{{#invoke:Language/scripts|showScripts|lá:yelhp|Quw̓utsun̓|Hul̓q̓umín̓um̓ / hən̓q̓əmin̓əm̓|xʷməθkʷəy̓əm|hən̓q̓əmin̓əm|Hul’q’umi’num’/Halq'eméyle/hən̓q̓əmin̓əm|sc̓əwaθən məsteyəxʷ|c̓əsnaʔəm}}

Lua error in package.lua at line 80: module 'Module:Utility' not found.


local p = {}

local gsub = mw.ustring.gsub
local length = mw.ustring.len
local floor = math.floor
local table_size = require("Module:TableTools").size
local UTF8Char = "[%z\1-\127\194-\244][\128-\191]*"

local data = require("Module:Language/scripts/data")

function p.print(frame)
	local scriptCode = frame.args[1]
	local scriptData = scriptCode and data[scriptCode] or "Please supply a valid script code."
	local characters = scriptData and scriptData.characters or "No characters found for " .. scriptCode .. "."
	return characters
end

local script = {}

-- Based on the Script:countCharacters() function of Module:scripts on Wiktionary
local function countCharacters(text, scriptCode)
	if not data[scriptCode]["characters"] then
		return 0
	else
		local _, count = gsub(text, "[" .. data[scriptCode]["characters"] .. "]", "")
		return count
	end
end

function p.isLatn(text)
	if type(tostring(text)) == "string" then
		local count = countCharacters(text, "Latn")
		if count < (length(text) / 4) then -- Only 25% of characters in string are Latin
			return false
		else
			return true
		end
	else
		return nil
	end
end

function p.Latin(frame)
	local text = frame.args[1]
	return p.isLatn(text)
end

local function map(func, t)
	local array = {}
	if t[1] then
		for i, v in ipairs(t) do
			array[i] = func(i, v, t)
		end
	else
		local i = 0
		for k, v in pairs(t) do
			i = i + 1
			array[i] = func(k, v, t)
		end
	end
	return array
end

local function sortRange(range1, range2)
	local number1, number2 = tonumber(range1[1]), tonumber(range2[1])
	if number1 == number2 then
		return keySort(range1[3], range2[3])
	else
		return number1 < number2
	end
end

--[[
	Binary search: more efficient for the longer lists of codepoint ranges than
	for the shorter ones.
]]
local function binarySearch(ranges, value)
	if not ranges then
		return nil
	end
	
	--	Initialize numbers.
	local bottom, middle = 1, 0
	-- Can't use # because table is loaded by mw.loadData.
	local top = table_size(ranges)

	if top == 0 then
		return nil
	end

	-- Do search.
	while bottom <= top do
		-- Calculate middle.
		middle = floor((bottom + top) / 2)

		-- Get compare value.
		local range = ranges[middle]

		if value < range[1] then
			top = middle - 1

		-- Return matching index. Assumes there are no duplicates.
		elseif value <= range[2] then
			return range

		-- Keep searching.
		else
			bottom = middle + 1
		end
	end
	return nil
end

local function lookUpInOrder(number, ranges)
	for i, range in ipairs(ranges) do
		if number < range[1] then
			return nil
		elseif number <= range[2] then
			return range[3]
		end
	end
end

-- Save previously used codepoint ranges in case another character is in the
-- same range.
local rangesCache = {}

--[=[
	Takes a codepoint and returns the script code that is appropriate for it,
	based on the data module [[Module:Language/scripts/codepoints]].
	
	The data module was copied from [[wikt:Module:Unicode data/scripts]], but
	Wiktionary-specific scripts have been removed. The Wiktionary module was
	generated from the patterns in [[wikt:Module:scripts/data]] using
	[[wikt:Module:User:Erutuon/script recognition]].

	Returns a script code if the codepoint is in the codepoint-to-script map, or
	in one of the ranges of 4096-codepoint groups or in one of the ranges of
	codepoints in the 4096-codepoint group that it belongs to, else returns
	"Zyyy" (the code for undetermined script).
]=]
function p.codepointToScript(codepoint)
	local lookup = mw.loadData("Module:language/scripts/codepoints")
	local t = type(codepoint)
	if t ~= "number" then
		error("Argument to codepointToScript should be a number, but its type is " .. t .. ".")
	end

	local individualMatch = lookup.individual[codepoint]
	if individualMatch then
		return individualMatch
	else
		local script = lookUpInOrder(codepoint, rangesCache)
		if script then
			return script
		end

		local index = floor(codepoint / 0x1000)

		script = lookUpInOrder(index, lookup.blocks)
		if script then
			return script
		end

		local range = binarySearch(lookup[index], codepoint)
		if range then
			table.insert(rangesCache, range)
			table.sort(rangesCache, sortRange)
			return range[3]
		end
	end

	return "Zyyy"
end

local function charToScript(char)
	return p.codepointToScript(mw.ustring.codepoint(char))
end

function p.countScripts(text)
	if type(text) ~= "string" then
		error("countScripts requires a string")
	end
	local scriptCounts = {}
	local codepointToScript = p.codepointToScript
	for codepoint in mw.ustring.gcodepoint(text) do
		local script = codepointToScript(codepoint)
		if script then
			if not scriptCounts[script] then
				scriptCounts[script] = 0
			end
			scriptCounts[script] = scriptCounts[script] + 1
		end
	end
	
	return scriptCounts
end

function p.showScripts(frame)
	return table.concat(
		map(function(i, arg)
				return "* " .. arg .. ": " .. table.concat(
					map(function(k, v)
							return k .. " (" .. v .. ")"
						end,
						p.countScripts(arg)),
					", ")
			end,
			frame.args),
		"\n")
end

local map = require("Module:Utility").map

local function sparseMap(func, array)
	local new_array = {}
	local new_i = 0
	for i, v in require("Module:TableTools").sparseIpairs(array) do
		new_i = new_i + 1
		new_array[new_i] = func(v, i)
	end
	return new_array
end

local function pre(text)
	return '<pre>' .. text .. '</pre>'
end

local function toHex(number)
	return ("0x%X"):format(number)
end

local function fromHex(str)
	return tonumber(str, 16)
end

function p.parseUnicodeScripts(frame)
	local scriptData = mw.title.new("User:Erutuon/Unicode scripts"):getContent():match("<!%-%-(.+)%-%->")
	
	local output = {}
	local ranges, individual = {}, {}
	local i = 1
	for line in scriptData:gmatch("\n%x[^\n]+") do
		local lower, higher, script = line:match("(%x+)%.%.(%x+) +; +(%a+)")
		if lower then
			ranges[i] = { fromHex(lower), fromHex(higher), script }
			i = i + 1
		else
			local codepoint, script = line:match("(%x+) +; +(%a+)")
			if codepoint then
				individual[fromHex(codepoint)] = script
			end
		end
		if i == 10 then
			break
		end
	end
	
	-- local output = scriptData:sub(1, 20)
	
	output = table.concat(
		map(
			function(range)
				return toHex(range[1]) .. "–" .. toHex(range[2]) .. ": " .. range[3]
			end,
			ranges),
		"\n")
		.. "\n" .. table.concat(
				sparseMap(
				function(script, codepoint)
					return toHex(codepoint) .. ": " .. script
				end,
				individual),
			"\n")
	
	return pre(output)
end

return p