Difference between revisions of "Module:Webarchive/sandbox"

From blackwiki
Jump to navigation Jump to search
blackwiki>Trappist the monk
(single point date formatting;)
blackwiki>Trappist the monk
Line 31: Line 31:
  
 
--[[--------------------------< G L O B A L  C O N F I G U R A T I O N  S E T T I N G S >--------------------
 
--[[--------------------------< G L O B A L  C O N F I G U R A T I O N  S E T T I N G S >--------------------
 +
 +
TODO: move these to ~/data
  
 
]]
 
]]
Line 42: Line 44:
  
 
Critical error. Render output completely in red. Add to tracking category.
 
Critical error. Render output completely in red. Add to tracking category.
 +
 +
TODO: move static text to ~/data
  
 
]]
 
]]
Line 74: Line 78:
 
Credit: https://de.wikipedia.org/wiki/Modul:Expr  
 
Credit: https://de.wikipedia.org/wiki/Modul:Expr  
  
TODO: have this function return nil with input is not wholly in the set %w
+
TODO: have this function return nil when input is not wholly in the set %w
  
 
]]
 
]]
Line 124: Line 128:
  
 
]]
 
]]
 
+
--[[
 
local function dateFormat(date)
 
local function dateFormat(date)
  
Line 146: Line 150:
 
return (y and (1900 < tonumber(y) and 2200 > tonumber(y))) and form; -- TODO: why 1900? shouldn't that be birth-of-intenet year?  why 2200? shouldn't that be current year?
 
return (y and (1900 < tonumber(y) and 2200 > tonumber(y))) and form; -- TODO: why 1900? shouldn't that be birth-of-intenet year?  why 2200? shouldn't that be current year?
 
end
 
end
 
+
]]
  
 
--[[--------------------------< D E C O D E _ D A T E >--------------------------------------------------------
 
--[[--------------------------< D E C O D E _ D A T E >--------------------------------------------------------
Line 153: Line 157:
 
names with proper capitalization, no error checking is done here; return nil else
 
names with proper capitalization, no error checking is done here; return nil else
  
TODO: modify to also return format of date_str?
 
 
]]
 
]]
  
 
local function decode_date (date_str)
 
local function decode_date (date_str)
 
local patterns = {
 
local patterns = {
{'^(%d%d?) +([^%s%d]+) +(%d%d%d%d)$', 'd', 'm', 'y'}, -- %a does not recognize unicode combining characters used by some languages
+
['dmy'] = {'^(%d%d?) +([^%s%d]+) +(%d%d%d%d)$', 'd', 'm', 'y'}, -- %a does not recognize unicode combining characters used by some languages
{'^([^%s%d]+) (%d%d?), +(%d%d%d%d)$', 'm', 'd', 'y'},
+
['mdy'] = {'^([^%s%d]+) (%d%d?), +(%d%d%d%d)$', 'm', 'd', 'y'},
{'^(%d%d%d%d) +([^%s%d]+) (%d%d?)$', 'y', 'm', 'd'}, -- not mos compliant at en.wiki but may be acceptible at other wikis
+
['ymd'] = {'^(%d%d%d%d) +([^%s%d]+) (%d%d?)$', 'y', 'm', 'd'}, -- not mos compliant at en.wiki but may be acceptible at other wikis
 
};
 
};
 
 
Line 170: Line 173:
 
 
 
if date_str:match ('^%d%d%d%d%-%d%d%-%d%d$') then -- already an iso format date, return western digits form
 
if date_str:match ('^%d%d%d%d%-%d%d%-%d%d$') then -- already an iso format date, return western digits form
return date_str;
+
return date_str, 'iso';
 
end
 
end
 
 
for i, v in ipairs (patterns) do
+
for k, v in pairs (patterns) do
local c1, c2, c3 = mw.ustring.match (date_str, patterns[i][1]); -- c1 .. c3 are captured but we don't know what they hold
+
local c1, c2, c3 = mw.ustring.match (date_str, patterns[k][1]); -- c1 .. c3 are captured but we don't know what they hold
 
 
 
if c1 then -- set on match
 
if c1 then -- set on match
 
t = { -- translate unspecified captures to y, m, and d
 
t = { -- translate unspecified captures to y, m, and d
[patterns[i][2]] = c1, -- fill the table of captures with the captures
+
[patterns[k][2]] = c1, -- fill the table of captures with the captures
[patterns[i][3]] = c2, -- take index names from src_pattern table and assign sequential captures
+
[patterns[k][3]] = c2, -- take index names from src_pattern table and assign sequential captures
[patterns[i][4]] = c3,
+
[patterns[k][4]] = c3,
 
};
 
};
 
if month_num[t.m] then -- when month not already a number
 
if month_num[t.m] then -- when month not already a number
Line 188: Line 191:
 
end
 
end
  
return mw.ustring.format ('%.4d-%.2d-%.2d', t.y, t.m, t.d); -- return date in iso format
+
return mw.ustring.format ('%.4d-%.2d-%.2d', t.y, t.m, t.d), k; -- return date in iso format
 
end
 
end
 
end
 
end
Line 196: Line 199:
 
--[[--------------------------< makeDate >-----------------------
 
--[[--------------------------< makeDate >-----------------------
  
Given a zero-padded 4-digit year, 2-digit month and 2-digit day, return a full date in df format
+
Given year, month, day nummbers, (zero-padded or not) return a full date in df format
df = mdy, dmy, iso, ymd
+
where df may be one of:
 +
mdy, dmy, iso, ymd
  
 
on entry, year, month, day are presumed to be correct for the date that they represent; all are required
 
on entry, year, month, day are presumed to be correct for the date that they represent; all are required
Line 211: Line 215:
 
};
 
};
  
local date = table.concat ({year, month, day}, '-'); -- assemble iso format date
+
local date = table.concat ({year, month, day}, '-'); -- assemble year-initial numeric-format date (zero padding not required here)
 
return mw.getContentLanguage():formatDate (format[df], date);
 
return mw.getContentLanguage():formatDate (format[df], date);
 
end
 
end
Line 354: Line 358:
  
 
-- snapdate = makeDate (year, month, day, df);
 
-- snapdate = makeDate (year, month, day, df);
snapdate = makeDate (year, month, day, 'iso'); -- date comparisons are all done in iso format
+
-- snapdate = makeDate (year, month, day, 'iso'); -- date comparisons are all done in iso format
 +
snapdate = table.concat ({year, month, day}, '-'); -- date comparisons are all done in iso format
 
if 14 == dlen then
 
if 14 == dlen then
 
return snapdate, msg; -- return date with message if any
 
return snapdate, msg; -- return date with message if any
Line 400: Line 405:
  
 
-- snapdate = makeDate(year, month, day, df);
 
-- snapdate = makeDate(year, month, day, df);
snapdate = makeDate (year, month, day, 'iso'); -- date comparisons are all done in iso format
+
-- snapdate = makeDate (year, month, day, 'iso'); -- date comparisons are all done in iso format
 +
snapdate = table.concat ({year, month, day}, '-'); -- date comparisons are all done in iso format
 
if 14 == dlen then
 
if 14 == dlen then
 
return snapdate; -- return date
 
return snapdate; -- return date
Line 748: Line 754:
 
date = date and date:gsub (' +', ' '); -- replace multiple spaces with a single space
 
date = date and date:gsub (' +', ' '); -- replace multiple spaces with a single space
 
if date and verifydates then
 
if date and verifydates then
ldf = dateFormat(date);
+
-- ldf = dateFormat(date);
date = decode_date(date);
+
date, ldf = decode_date(date);
 
end
 
end
  
Line 851: Line 857:
 
});
 
});
 
-- date = udate .. inlineRed (err_warn_msgs.mismatch, 'warning') .. (msg or ''); -- mismatch: use url date; add message if there is one
 
-- date = udate .. inlineRed (err_warn_msgs.mismatch, 'warning') .. (msg or ''); -- mismatch: use url date; add message if there is one
else
+
-- else
date = makeDate (date, nil, nil, ldf); -- TODO: makeDate() once at the end -- date already in iso format so spoof makeDate()
+
-- date = makeDate (date, nil, nil, ldf); -- TODO: makeDate() once at the end -- date already in iso format so spoof makeDate()
 
-- date = date .. (msg or ''); -- add message if there is one
 
-- date = date .. (msg or ''); -- add message if there is one
 
end
 
end

Revision as of 15:32, 19 September 2018

Documentation for this module may be created at Module:Webarchive/sandbox/doc

--[[ ----------------------------------

Lua module implementing the {{webarchive}} template. 

A merger of the functionality of three templates: {{wayback}}, {{webcite}} and {{cite archives}}
	
]]

require('Module:No globals');
local getArgs = require ('Module:Arguments').getArgs;
local this_page = mw.title.getCurrentTitle();


--[[--------------------------< F O R W A R D   D E C L A R A T I O N S >--------------------------------------
]]

local categories = {};															-- category names from ./data
local err_warn_msgs = {};														-- error and warning messages from ./data
local excepted_pages = {};
local prefixes = {};															-- service provider tail string prefixes from ./data
local services = {};															-- archive service provider data from ./data
local uncategorized_namespaces = {};											-- list of namespaces that we should not categorize
local uncategorized_subpages = {};												-- list of subpages that should not be categorized

local month_num = {};															-- table that translates local-wiki month names to western digits
local digits = {};																-- table that translates local-wiki digits to western digits

local ulx = {};																	-- Associative array to hold template data 
local track = {};																-- Associative array to hold tracking categories


--[[--------------------------< G L O B A L   C O N F I G U R A T I O N   S E T T I N G S >--------------------

TODO: move these to ~/data

]]

local maxurls = 10;																-- Max number of URLs allowed. 
local tname = 'Webarchive'														-- name of calling template. Change if template rename.
local verifydates = 'yes'														-- See documentation. Set "no" to disable.


--[[--------------------------< inlineError >-----------------------

Critical error. Render output completely in red. Add to tracking category.

TODO: move static text to ~/data

]]

local function inlineError(arg, msg)
	track[categories.error] = 1
	return '<span style="font-size:100%" class="error citation-comment">Error in webarchive template: Check <code style="color:inherit; border:inherit; padding:inherit;">&#124;' .. arg .. '=</code> value. ' .. msg .. '</span>'
end


--[[--------------------------< inlineRed >-----------------------

Render a text fragment in red, such as a warning as part of the final output.
Add tracking category.

 ]]

local function inlineRed(msg, trackmsg)
	if trackmsg == "warning" then
		track[categories.warning] = 1;
	elseif trackmsg == "error" then
		track[categories.error] = 1;
	end

	return '<span style="font-size:100%" class="error citation-comment">' .. msg .. '</span>'
end


--[[--------------------------< base62 >-----------------------

Convert base-62 to base-10
Credit: https://de.wikipedia.org/wiki/Modul:Expr 

TODO: have this function return nil when input is not wholly in the set %w

]]

local function base62( value )
	local r = 1																	-- default return value is input value is malformed

	if value:match( "^%w+$" ) then												-- value must only be in the set [0-9a-zA-Z]
		local n = #value														-- number of characters in value
		local k = 1
		local c
		r = 0
		for i = n, 1, -1 do														-- loop through all characters in value from ls digit to ms digit
			c = value:byte( i, i )
			if c >= 48 and c <= 57 then											-- character is digit 0-9
				c = c - 48
			elseif c >= 65 and c <= 90 then										-- character is ascii a-z
				c = c - 55
			else																-- must be ascii A-Z
				c = c - 61
			end
			r = r + c * k														-- accumulate this base62 character's value
			k = k * 62															-- bump for next
		end -- for i
	end
	return r
end 


--[[--------------------------< tableLength >-----------------------

Given a 1-D table, return number of elements

]]

local function tableLength(T)
	local count = 0
	for _ in pairs(T) do count = count + 1 end
	return count
end


--[[--------------------------< dateFormat >-----------------------

Given a date string, return its format: dmy, mdy, iso, ymd
If unable to determine return nil

TODO: merge this functionality into decode_date()?

]]
--[[
local function dateFormat(date)

	local patterns = {
		['iso'] = '(%d%d%d%d)%-%d%d%-%d%d',
		['dmy'] = '%d%d? +%a+ +(%d%d%d%d)',
		['mdy'] = '%a+ %d%d?, +(%d%d%d%d)',
		['ymd'] = '(%d%d%d%d) +%a+ %d%d?',										-- not mos compliant at en.wiki but may be acceptible at other wikis
		};

	local form,  y;

	for k, v in pairs (patterns) do												-- loop through the patterns table
		y = mw.ustring.match (date, v);											-- looking for a match
		if y then																-- not nil when found
			form = k;															-- save that
			break;																-- and done
		end
	end

	return (y and (1900 < tonumber(y) and 2200 > tonumber(y))) and form;		-- TODO: why 1900? shouldn't that be birth-of-intenet year?  why 2200? shouldn't that be current year?
end
]]

--[[--------------------------< D E C O D E _ D A T E >--------------------------------------------------------

Given a date string, return it in iso format.  Except that month names must be recognizable as legitimate month
names with proper capitalization, no error checking is done here; return nil else

]]

local function decode_date (date_str)
	local patterns = {
		['dmy'] = {'^(%d%d?) +([^%s%d]+) +(%d%d%d%d)$', 'd', 'm', 'y'},					-- %a does not recognize unicode combining characters used by some languages
		['mdy'] = {'^([^%s%d]+) (%d%d?), +(%d%d%d%d)$', 'm', 'd', 'y'},
		['ymd'] = {'^(%d%d%d%d) +([^%s%d]+) (%d%d?)$', 'y', 'm', 'd'},					-- not mos compliant at en.wiki but may be acceptible at other wikis
		};
	
	local t = {};
	
	if 0 ~= tableLength (digits) then											-- must be filled for wikis that do not use western digits
		date_str = mw.ustring.gsub (date_str, '%d', digits);					-- convert this wiki's non-western digits to western digits
	end
	
	if date_str:match ('^%d%d%d%d%-%d%d%-%d%d$') then							-- already an iso format date, return western digits form
		return date_str, 'iso';
	end
	
	for k, v in pairs (patterns) do
		local c1, c2, c3 = mw.ustring.match (date_str, patterns[k][1]);			-- c1 .. c3 are captured but we don't know what they hold
		
		if c1 then																-- set on match
			t = {																-- translate unspecified captures to y, m, and d
				[patterns[k][2]] = c1,											-- fill the table of captures with the captures
				[patterns[k][3]] = c2,											-- take index names from src_pattern table and assign sequential captures
				[patterns[k][4]] = c3,
				};
			if month_num[t.m] then												-- when month not already a number
				t.m = month_num[t.m];											-- replace valid month name with a number
			else
				return nil;														-- not a valid date form because month not valid
			end

			return mw.ustring.format ('%.4d-%.2d-%.2d', t.y, t.m, t.d), k;		-- return date in iso format
		end
	end
end

	
--[[--------------------------< makeDate >-----------------------

Given year, month, day nummbers, (zero-padded or not) return a full date in df format
where df may be one of:
	mdy, dmy, iso, ymd

on entry, year, month, day are presumed to be correct for the date that they represent; all are required

]]

local function makeDate (year, month, day, df)
	local format = {
		['dmy'] = 'j F Y',
		['mdy'] = 'F j, Y',
		['ymd'] = 'Y F j',
		['iso'] = 'Y-m-d',
		};

	local date = table.concat ({year, month, day}, '-');						-- assemble year-initial numeric-format date (zero padding not required here)
	return mw.getContentLanguage():formatDate (format[df], date);
end


--[[--------------------------< I S _ V A L I D _ D A T E >----------------------------------------------------

Returns true if date is after 31 December 1899 (why is 1900 the min year? shouldn't the internet's date-of-birth
be min year?), not after today's date, and represents a valid date (29 February 2017 is not a valid date).  Applies
Gregorian leapyear rules.

all arguments are required

]]

local function is_valid_date (year, month, day)
	local days_in_month = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31};
	local month_length;
	local y, m, d;
	local today = os.date ('*t');												-- fetch a table of current date parts

	if not year or '' == year or not month or '' == month or not day or '' == day then
		return false;															-- something missing
	end
	
	y = tonumber (year);
	m = tonumber (month);
	d = tonumber (day);

	if 1900 > y or today.year < y or 1 > m or 12 < m then						-- year and month are within bounds	TODO: 1900?
		return false;
	end

	if (2==m) then																-- if February
		month_length = 28;														-- then 28 days unless
		if (0==(y%4) and (0~=(y%100) or 0==(y%400))) then						-- is a leap year?
			month_length = 29;													-- if leap year then 29 days in February
		end
	else
		month_length=days_in_month[m];
	end

	if 1 > d or month_length < d then											-- day is within bounds
		return false;
	end
																					-- here when date parts represent a valid date
	return os.time({['year']=y, ['month']=m, ['day']=d, ['hour']=0}) <= os.time();	-- date at midnight must be less than or equal to current date/time
end


--[[--------------------------< decodeWebciteDate >-----------------------

Given a URI-path to Webcite (eg. /67xHmVFWP) return the encoded date in df format

returns date string in df format - webcite date is a unix timestamp encoded as bae62
or the string 'query'

TODO: modify base62() so that it returns nil when the value sent to it is not in the set %w

]]

local function decodeWebciteDate(path, df)

	local dt = {}

	dt = mw.text.split(path, "/")

	-- valid URL formats that are not base62

	-- http://www.webcitation.org/query?id=1138911916587475
	-- http://www.webcitation.org/query?url=http..&date=2012-06-01+21:40:03
	-- http://www.webcitation.org/1138911916587475
	-- http://www.webcitation.org/cache/73e53dd1f16cf8c5da298418d2a6e452870cf50e
	-- http://www.webcitation.org/getfile.php?fileid=1c46e791d68e89e12d0c2532cc3cf629b8bc8c8e

	if dt[2]:find ('query', 1, true) or 
		dt[2]:find ('cache', 1, true) or
		dt[2]:find ('getfile', 1, true) or
		tonumber(dt[2]) then
			return 'query';
	end

	dt = os.date('*t', string.format("%d", base62(dt[2])):sub(1,10))			-- base62 string -> exponential number -> text -> first 10 characters (a unix timestamp) -> a table of date parts

--	return makeDate(dt.year, dt.month, dt.day, df);
	return makeDate(dt.year, dt.month, dt.day, 'iso');							-- date comparisons are all done in iso format
end


--[[--------------------------< decodeWaybackDate >-----------------------

Given a URI-path to Wayback (eg. /web/20160901010101/http://example.com )
or Library of Congress Web Archives (/all/20160901010101/http://example.com)
return the formatted date eg. "September 1, 2016" in df format 
Handle non-digits in snapshot ID such as "re_" and "-" and "*"

returns two values:
	first value is one of these:
		valid date string in df format - wayback date is valid (including the text string 'index' when date is '/*/')
		empty string - wayback date is malformed (not a number, less than 8 digits, not a valid date)
		nil - wayback date is '/save/'
	
	second return value is an appropriate 'message' may or may not be formatted

]]

local function decodeWaybackDate(path, df)

	local msg, snapdate;

	if path:match ('^/save/') then												-- fatal error; don't allow rendering of this url link because it saves an ew copy of target page when clicked
		return nil, 'save';														-- return nil (fatal error flag) and message selector
	end

	snapdate = path:gsub ('^/all/', ''):gsub ('^/web/', ''):gsub ('^/', '');	-- remove leading '/all/', leading '/web/' or leading '/'
	snapdate = snapdate:match ('^[^/]+');										-- get timestamp
	if snapdate == "*" then														-- eg. /web/*/http.. or /all/*/http..
		return 'index';															-- return indicator that this url has an index date
	end

	snapdate = snapdate:gsub ('%a%a_%d?$', ''):gsub ('%-', '');					-- from date, remove any trailing "re_", dashes

	msg = '';
	if snapdate:match ('%*$') then												-- a trailing '*' causes calendar display at archive .org
		snapdate = snapdate:gsub ('%*$', '');									-- remove so not part of length calc later
		msg = inlineRed (err_warn_msgs.ts_cal, 'warning');						-- make a message
	end

	if not tonumber(snapdate) then
		return '', inlineRed (err_warn_msgs.ts_nan, 'error');					-- return empty string and error message
	end
	local dlen = snapdate:len();
	if dlen < 8 then															-- we need 8 digits TODO: but shouldn't this be testing for 14 digits?
		return '', inlineRed (err_warn_msgs.ts_short, 'error');					-- return empty string and error message
	end

	local year, month, day = snapdate:match ('(%d%d%d%d)(%d%d)(%d%d)');			-- no need for snapdatelong here

	if not is_valid_date (year, month, day) then
		return '', inlineRed (err_warn_msgs.ts_date, 'error');					-- return empty string and error message
	end

--	snapdate = makeDate (year, month, day, df);
--	snapdate = makeDate (year, month, day, 'iso');								-- date comparisons are all done in iso format
	snapdate = table.concat ({year, month, day}, '-');							-- date comparisons are all done in iso format
	if 14 == dlen then
		return snapdate, msg;													-- return date with message if any
	else
		return snapdate, msg .. inlineRed (err_warn_msgs.ts_len, 'warning');	-- return date with warning message(s)
	end
end


--[[--------------------------< decodeArchiveisDate >-----------------------

Given an Archive.is "long link" URI-path (e.g. /2016.08.28-144552/http://example.com)
return the date in df format (e.g. if df = dmy, return 28 August 2016)
Handles "." and "-" in snapshot date, so 2016.08.28-144552 is same as 20160828144552

returns two values:
	first value is one of these:
		valid date string in df format - archive.is date is valid (including the text string 'short link' when url is the short form)
		empty string - wayback date is malformed (not a number, less than 8 digits, not a valid date)
		nil - wayback date is '/save/'
	
	second return value is an appropriate 'message' may or may not be formatted

]]

local function decodeArchiveisDate(path, df)
	local snapdate

	if path:match ('^/%w+$') then												-- short form url path is '/' followed by some number of base 62 digits and nothing else
		return "short link"														-- e.g. http://archive.is/hD1qz
	end

	snapdate = mw.text.split (path, '/')[2]:gsub('[%.%-]', '');					-- get snapshot date, e.g. 2016.08.28-144552; remove periods and hyphens

	local dlen = string.len(snapdate)
	if dlen < 8 then															-- we need 8 digits TODO: but shouldn't this be testing for 14 digits?
		return '', inlineRed (err_warn_msgs.ts_short, 'error');					-- return empty string and error message
	end

	local year, month, day = snapdate:match ('(%d%d%d%d)(%d%d)(%d%d)');			-- no need for snapdatelong here

	if not is_valid_date (year, month, day) then
		return '', inlineRed (err_warn_msgs.ts_date, 'error');					-- return empty string and error message
	end

--	snapdate = makeDate(year, month, day, df);
--	snapdate = makeDate (year, month, day, 'iso');								-- date comparisons are all done in iso format
	snapdate = table.concat ({year, month, day}, '-');							-- date comparisons are all done in iso format
	if 14 == dlen then
		return snapdate;														-- return date
	else
		return snapdate, inlineRed (err_warn_msgs.ts_len, 'warning');			-- return date with warning message
	end
 end


--[=[-------------------------< M A K E _ W I K I L I N K >----------------------------------------------------

Makes a wikilink; when both link and display text is provided, returns a wikilink in the form [[L|D]]; if only
link is provided, returns a wikilink in the form [[L]]; if neither are provided or link is omitted, returns an
empty string.

]=]

local function make_wikilink (link, display, no_link)
	if nil == no_link then
		if link and ('' ~= link) then
			if display and ('' ~= display) then
				return table.concat ({'[[', link, '|', display, ']]'});
			else
				return table.concat ({'[[', link, ']]'});
			end
		end
		return display or '';													-- link not set so return the display text

	else																		-- no_link
		if display and ('' ~= display) then										-- if there is display text
			return display;														-- return that
		else
			return link or '';													-- return the target article name or empty string
		end
	end
end


--[[--------------------------< serviceName >-----------------------

Given a domain extracted by mw.uri.new() (eg. web.archive.org) set tail string and service ID

]]

local function serviceName(host, no_link)
	local tracking;
	local index;
	
	host = host:lower():gsub ('^web%.(.+)', '%1'):gsub ('^www%.(.+)', '%1');	-- lowercase, remove web. and www. subdomains

	if services[host] then
		index = host;
	else
		for k, _ in pairs (services) do
			if host:find ('%f[%a]'..k:gsub ('([%.%-])', '%%%1')) then
				index = k;
				break;
			end
		end
	end
	
	if index then
		local out = {''};														-- empty string in [1] so that concatenated result has leading single space
		ulx.url1.service = services[index][4] or 'other';
		tracking = services[index][5] or categories.other;
																				-- build tail string
		if false == services[index][1] then										-- select prefix
			table.insert (out, prefixes.at);
		elseif true == services[index][1] then
			table.insert (out, prefixes.atthe);
		else
			table.insert (out, services[index][1]);
		end
		
		table.insert (out, make_wikilink (services[index][2], services[index][3], no_link));	-- add article wikilink
		if services[index][6] then												-- add tail postfix if it exists
			table.insert (out, services[index][6]);
		end
		
		ulx.url1.tail = table.concat (out, ' ');								-- put it all together; result has leading space character

	else																		-- here when unknown archive
		ulx.url1.service = 'other';
		tracking = categories.unknown;
		ulx.url1.tail = table.concat ({'', prefixes.at, host, inlineRed (err_warn_msgs.unknown_url)}, ' ');	-- TODO: this call to inlineRed() does not specify 'error' or 'warning'; should it?
	end
	
	track[tracking] = 1
end


--[[--------------------------< parseExtraArgs >-----------------------

Parse numbered arguments starting at 2, such as url2..url10, date2..date10, title2..title10
	For example: {{webarchive |url=.. |url4=.. |url7=..}}
		Three url arguments not in numeric sequence (1..4..7). 
			Function only processes arguments numbered 2 or greater (in this case 4 and 7)
				It creates numeric sequenced table entries like:
				urlx.url2.url = <argument value for url4>
				urlx.url3.url = <argument value for url7>
			Returns the number of URL arguments found numbered 2 or greater (in this case returns "2")

 ]]

local function parseExtraArgs(args)

	local i, j, argurl, argurl2, argdate, argtitle

	j = 2
	for i = 2, maxurls do
		argurl = "url" .. i
		if args[argurl] then
			argurl2 = "url" .. j
			ulx[argurl2] = {}
			ulx[argurl2]["url"] = args[argurl]
			argdate = "date" .. j
			if args[argdate] then
				ulx[argurl2]["date"] = args[argdate]
			else
				ulx[argurl2]["date"] = inlineRed (err_warn_msgs.date_miss, 'warning');
			end
	
			argtitle = "title" .. j
			if args[argtitle] then
				ulx[argurl2]["title"] = args[argtitle]
			else
				ulx[argurl2]["title"] = nil
			end
			j = j + 1
		end
	end

	if j == 2 then
		return 0
	else
		return j - 2
	end
end


--[[--------------------------< comma >-----------------------

Given a date string, return "," if it's MDY 

]]

local function comma(date)
	return (date and date:match ('%a+ +%d%d?(,) +%d%d%d%d')) or '';
end


--[[--------------------------< createTracking >-----------------------

Return data in track[] ie. tracking categories

]]

local function createTracking()
	if not excepted_pages[this_page.fullText] then								-- namespace:title/fragment is allowed to be categorized (typically this module's / template's testcases page(s))
		if uncategorized_namespaces[this_page.nsText] then
			return '';															-- this page not to be categorized so return empty string
		end
		for _,v in ipairs (uncategorized_subpages) do							-- cycle through page name patterns
			if this_page.text:match (v) then									-- test page name against each pattern
				return '';														-- this subpage type not to be categorized so return empty string
			end
		end
	end

	local out = {};
	if tableLength(track) > 0 then
		for key, _ in pairs(track) do											-- loop through table
			table.insert (out, make_wikilink (key));							-- and convert category names to links
		end
	end
	return table.concat (out);													-- concat into one big string; empty string if table is empty

end


--[[--------------------------< createRendering >-----------------------

Return a rendering of the data in ulx[][]

]]

local function createRendering()

	local displayfield
	local out = {};
	
	local period1 = '';															-- For backwards compat with {{wayback}}
	local period2 = '.';															

	if 'none' == ulx.url1.format then											-- For {{wayback}}, {{webcite}}
		table.insert (out, '[');												-- open extlink markup
		table.insert (out, ulx.url1.url);										-- add url

		if ulx.url1.title then
			table.insert (out, ' ')												-- the required space
			table.insert (out, ulx.url1.title)									-- the title
			table.insert (out, ']');											-- close extlink markup
			table.insert (out, ulx.url1.tail);									-- tail text
			if ulx.url1.date then
				table.insert (out, '&#32;(');									-- open date text; TODO: why the html entity?
				table.insert (out, 'index' == ulx.url1.date and 'archive' or 'archived');	-- add text
				table.insert (out, ' ');										-- insert a space
				table.insert (out, ulx.url1.date);								-- add date
				table.insert (out, ')');										-- close date text
			end
		else																	-- no title
			table.insert (out, 'index' == ulx.url1.date and ' Archive index]' or ' Archived] ');	-- begin link label -- use this line for correct link label when date is 'index'
			if ulx.url1.date then
				if 'wayback' == ulx.url1.service then 
					period1 = '.';
					period2 = '';
				end
				if 'index' ~= ulx.url1.date then
					table.insert (out, ulx.url1.date);							-- add date when data is not 'index'
				end
				table.insert (out, comma(ulx.url1.date));						-- add ',' if date format is mdy
				table.insert (out, ulx.url1.tail);								-- add tail text
				table.insert (out, period1);									-- terminate
			else																-- no date
				table.insert (out, ulx.url1.tail);								-- add tail text
			end
		end

		if 0 < ulx.url1.extraurls then											-- For multiple archive URLs
			local tot = ulx.url1.extraurls + 1
			table.insert (out, period2);										-- terminate first url
			table.insert (out, ' Additional archives: ');						-- add header text

			for i=2, tot do														-- loop through the additionals
				local index = table.concat ({'url', i});						-- make an index
				displayfield = ulx[index]['title'] and 'title' or 'date';		-- choose display text
				table.insert (out, '[');										-- open extlink markup
				table.insert (out, ulx[index]['url']);							-- add the url
				table.insert (out, ' ');										-- the required space
				table.insert (out, ulx[index][displayfield]);					-- add the label
				table.insert (out, ']');										-- close extlink markup
				table.insert (out, i==tot and '.' or ', ');						-- add terminator
			end
		end
		return table.concat (out);												-- make a big string and done

	else																		-- For {{cite archives}}																	
		if 'addlarchives' == ulx.url1.format then								-- Multiple archive services 
			table.insert (out, 'Additional archives: ');						-- add header text
		else																	-- Multiple pages from the same archive 
			table.insert (out, 'Additional pages archived&nbsp;on ');			-- add header text
			table.insert (out, ulx.url1.date);									-- add date to header text
			table.insert (out, ': ');											-- close header text
		end

		local tot = ulx.url1.extraurls + 1;
		for i=1, tot do															-- loop through the additionals
			local index = table.concat ({'url', i});							-- make an index
			table.insert (out, '[');											-- open extlink markup
			table.insert (out, ulx[index]['url']);								-- add url
			table.insert (out, ' ');											-- add required space

			displayfield = ulx[index]['title'];
			if 'addlarchives' == ulx.url1.format then
				if not displayfield then 
					displayfield = ulx[index]['date']
				end
			else																-- must be addlpages
				if not displayfield then 
					displayfield = table.concat ({'Page ', i});
				end
			end
			table.insert (out, displayfield);									-- add title, date, page label text
			table.insert (out, ']');											-- close extlink markup
			table.insert (out, (i==tot and '.' or ', '));							-- add terminator
		end
		return table.concat (out);												-- make a big string and done
	end
end


--[[--------------------------< W E B A R C H I V E >----------------------------------------------------------

template entry point

]]

local function webarchive(frame)
	local args = getArgs (frame);
	
	local data = mw.loadData (table.concat ({									-- make a data module name; sandbox or live
		'Module:Webarchive/data',
		frame:getTitle():find('sandbox', 1, true) and '/sandbox' or ''			-- this instance is ./sandbox then append /sandbox
		}));
	categories = data.categories;												-- fill in the forward declarations
	err_warn_msgs = data.err_warn_msgs;
	excepted_pages = data.excepted_pages;
	prefixes = data.prefixes;
	services = data.services;
	uncategorized_namespaces = data.uncategorized_namespaces;
	uncategorized_subpages = data.uncategorized_subpages;
	month_num = data.month_num;
	digits = data.digits;
	

--	local date, format, msg, uri, url;
	local date, format, ldf, msg, udate, uri, url;
	
	verifydates = 'yes' == verifydates;											-- convert to boolean

	if args.url and args.url1 then												-- URL argument (first)
		return inlineError("url", "Conflicting |url= and |url1=.") .. createTracking();
	end
	
	url = args.url or args.url1;
	
	if not url then
		return inlineError("url", "Empty.") .. createTracking()
	end
																				-- these iabot bugs perportedly fixed; removing these causes lua script error
																				-- at Template:Webarchive/testcases/Production; resolve that before deleting these tests
	if mw.ustring.find( url, "https://web.http", 1, true ) then					-- track bug - TODO: IAbot bug; not known if the bug has been fixed; deferred
		track[categories.error] = 1;
		return inlineError("url", "https://web.http") .. createTracking()
	end 
	if url == "https://web.archive.org/http:/" then								 -- track bug - TODO: IAbot bug; not known if the bug has been fixed; deferred
		track[categories.error] = 1;
		return inlineError("url", "Invalid URL") .. createTracking()
	end

	ulx.url1 = {}
	ulx.url1.url = url
	if not (url:lower():find ('^http') or url:find ('^//')) then				-- TODO: is this a good idea? isn't it better to simply throw an error when url is malformed ...
		ulx.url1.url = 'http://' .. url											-- ... rather than apply this 'fix' that might not fix anything?
	end

	ulx.url1.extraurls = parseExtraArgs(args)

	uri = mw.uri.new (ulx.url1.url);											-- get a table of uri parts from this url
	serviceName(uri.host, args.nolink)

	if args.date and args.date1 then											-- Date argument
		return inlineError("date", "Conflicting |date= and |date1=.") .. createTracking();
	end
	
	date = args.date or args.date1;
	date = date and date:gsub (' +', ' ');										-- replace multiple spaces with a single space
if date and verifydates then
--	ldf = dateFormat(date);
	date, ldf = decode_date(date);
end

	if 'wayback' == ulx.url1.service or 'locwebarchives' == ulx.url1.service then
		if '*' == date then														-- TODO: why is this not compared to url date?
			date = 'index';
		end
		if date then
			if verifydates then
--				local ldf = dateFormat(date)
				if ldf then
					udate, msg = decodeWaybackDate( uri.path, ldf )		-- TODO: return iso date for comparison instead of multi formats -- get the url date in the same format as date in |date=; 'index' when wayback date is *
--					local udate, msg = decodeWaybackDate( uri.path, ldf )		-- TODO: return iso date for comparison instead of multi formats -- get the url date in the same format as date in |date=; 'index' when wayback date is *
					if not udate then											-- this is the only 'fatal' error return
						return inlineError("url", "Invalid url: /save/") .. createTracking();
					end

--					if udate ~= date then										-- also when decodeWaybackDate() return ''
--					if decode_date(udate) ~= decode_date(date) then				-- date comparison using iso format dates
					if udate ~= date then							-- date comparison using iso format dates
						date = udate;
						msg = table.concat ({
--							makeDate (udate, nil, nil, ldf),					-- TODO: makeDate() once at the end					-- udate already in iso format so spoof makeDate()
							inlineRed (err_warn_msgs.mismatch, 'warning'),		-- add warning message
							msg,												-- add message if there is one
						});
--						date = udate .. inlineRed (err_warn_msgs.mismatch, 'warning') .. (msg or '');	-- mismatch use url date; add message if there is one
--					else
--						date = makeDate (date, nil, nil, ldf)					-- TODO: makeDate() once at the end -- date already in iso format so spoof makeDate()
--						date = date .. (msg or '');								-- add message if there is one
					end
				end
			end
		else																	-- no |date=
			ldf = 'iso';
			udate, msg = decodeWaybackDate( uri.path, "iso" )
			if not udate then													-- this is the only 'fatal' error return
				return inlineError("url", "Invalid url: /save/") .. createTracking();
			end

			if '' == udate then 
				date = nil;														-- unset
--				date = msg;														-- date couldn't be decoded; use message
			else
				date = udate;
--				date = date .. (msg or '');										-- date could be decoded; add message if there is one
			end
		end

	elseif 'webcite' == ulx.url1.service then
		if date then
			if verifydates then
--				local ldf = dateFormat(date)
				if ldf then
					udate = decodeWebciteDate( uri.path, ldf )			-- get the url date in the same format as date in |date=
--					local udate = decodeWebciteDate( uri.path, ldf )			-- get the url date in the same format as date in |date=
					if 'query' ~= udate then									-- skip if query
--						if udate ~= date then
--						if decode_date(udate) ~= decode_date(date) then			-- date comparison using iso format dates
						if udate ~= date then						-- date comparison using iso format dates
							date = udate;
							msg = table.concat ({
--								makeDate (udate, nil, nil, ldf),				-- TODO: makeDate() once at the end
								inlineRed (err_warn_msgs.mismatch, 'warning'),
								});
--							date = udate .. inlineRed (err_warn_msgs.mismatch, 'warning');
--						else
--							date = makeDate (udate, nil, nil, ldf);				-- TODO: makeDate() once at the end
						end
					end
				end
			end
		else
			ldf = 'iso';
			date = decodeWebciteDate( uri.path, "iso" )
			if date == "query" then
				date = nil;														-- unset
				msg = inlineRed (err_warn_msgs.date_miss, 'warning');
--				date = inlineRed (err_warn_msgs.date_miss, 'warning');
--			elseif not date then												-- pointless for now because base62() always returns a number regardless of its input TODO: fix that
--				date = inlineRed (err_warn_msgs.date1, 'error');
			end
		end

	elseif 'archiveis' == ulx.url1.service then
		if date then
			if verifydates then
--				local ldf = dateFormat(date)
				if ldf then
					udate, msg = decodeArchiveisDate( uri.path, ldf )			-- get the url date in the same format as date in |date=
--					local udate, msg = decodeArchiveisDate( uri.path, ldf )			-- get the url date in the same format as date in |date=
					if 'short link' ~= udate then								-- skip if short link
--						if udate ~= date then
--						if decode_date(udate) ~= decode_date(date) then			-- date comparison using iso format dates
						if udate ~= date then						-- date comparison using iso format dates
							date = udate;
							msg = table.concat ({
--								makeDate (udate, nil, nil, ldf),				-- TODO: makeDate() once at the end					-- udate already in iso format so spoof makeDate()
								inlineRed (err_warn_msgs.mismatch, 'warning'),	-- add warning message
								msg,											-- add message if there is one
							});
--							date = udate .. inlineRed (err_warn_msgs.mismatch, 'warning') .. (msg or '');	-- mismatch: use url date; add message if there is one
--						else
--							date = makeDate (date, nil, nil, ldf);				-- TODO: makeDate() once at the end -- date already in iso format so spoof makeDate()
--							date = date .. (msg or '');							-- add message if there is one
						end
--					else
--						date = makeDate (date, nil, nil, ldf);					-- TODO: makeDate() once at the end -- date already in iso format so spoof makeDate()
					end
				end
			end
		else																	-- no |date=
			ldf = 'iso';
			udate, msg = decodeArchiveisDate( uri.path, "iso" )
			if udate == "short link" then
				date = nil;														-- unset
				msg = inlineRed (err_warn_msgs.date_miss, 'warning');
--				date = inlineRed (err_warn_msgs.date_miss, 'warning');
			elseif '' == udate then
				date = nil;														-- unset
--				date = msg;														-- date couldn't be decoded; use message
			else
				date = udate;
--				date = date .. (msg or '');										-- date could be decoded; add message if there is one
			end
		end
		
	else																		-- some other service
		if not date then
			msg = inlineRed (err_warn_msgs.date_miss, 'warning');
--			date = inlineRed (err_warn_msgs.date_miss, 'warning');
		end
	end

--	ulx.url1.date = date
	if 'index' == date then
		ulx.url1.date = date;
	elseif date then
		ulx.url1.date = makeDate (date, nil, nil, ldf) .. (msg or '');
	else
		ulx.url1.date = msg;
	end

	format = args.format;														-- Format argument 

	if not format then
		format = "none"
	else
		if format == "addlpages" then
			if not ulx.url1.date then
				format = "none"
			end
		elseif format == "addlarchives" then
			format = "addlarchives"
		else
			format = "none"
		end
	end
	ulx.url1.format = format

	if args.title and args.title1 then											-- Title argument
		return inlineError("title", "Conflicting |title= and |title1=.") .. createTracking();
	end

	ulx.url1.title = args.title or args.title1;

	local rend = createRendering()
	if not rend then
		rend = '<span style="font-size:100%" class="error citation-comment">Error in [[:Template:' .. tname .. ']]: Unknown problem. Please report on template talk page.</span>'
		track[categories.error] = 1;
	end

	return rend .. createTracking()
end


--[[--------------------------< E X P O R T E D 	 F U N C T I O N S >------------------------------------------
]]

return {webarchive = webarchive};