basic epwing/json dictionary migration and management done

This commit is contained in:
wareya committed 2019-05-13 20:01:11 -04:00
1 parent 61bce7a3bb
commit 2a69981d6e
8 files changed
+336 -203

No files matched your search

+136 -70
View File
@@ -48,49 +48,86 @@ browser.storage.onChanged.addListener((updates, storageArea) =>
// We have to send an HTTP request and store the result in a string before parsing it into an object.
// Seriously.
let epwing = [];
let lookup_epwing_kan = new Map();
let lookup_epwing_kana = new Map();
let json_dicts = [];
async function refresh_epwing()
function add_json_reading(dict, text, id)
{
let myjson = (await browser.storage.local.get("epwing"))["epwing"];
if(text === "")
return;
if (!dict.lookup_json_kana.has(text))
dict.lookup_json_kana.set(text, [id]);
else
dict.lookup_json_kana.get(text).push(id);
}
function add_json_spelling(dict, text, id)
{
if(text === "")
return;
if (!dict.lookup_json_kan.has(text))
dict.lookup_json_kan.set(text, [id]);
else
dict.lookup_json_kan.get(text).push(id);
}
function or_default(val, fallback)
{
if(val !== undefined)
return val;
else
return fallback;
}
async function get_storage_or_default(name, fallback)
{
return or_default((await browser.storage.local.get(name))[name], fallback);
}
async function refresh_json()
{
let custom_dicts = (await get_storage_or_default("custom_dicts", []));
json_dicts = [];
for(let stored_dict of custom_dicts)
{
if(!stored_dict.enabled)
continue;
let name = stored_dict.name;
let entries = JSON.parse(stored_dict.entries);
let lookup_json_kan = new Map();
let lookup_json_kana = new Map();
let dict = {"name": name, "entries": entries, "lookup_json_kan": lookup_json_kan, "lookup_json_kana": lookup_json_kana};
try
{
for (let i = 0; i < entries.length; i++)
{
add_json_reading(dict, entries[i]["r"], i);
for(let spelling of entries[i]["s"])
add_json_spelling(dict, spelling, i);
}
} catch(e){}
json_dicts.push(dict);
}
}
async function migrate_legacy_then_refresh()
{
let myjson = (await browser.storage.local.get(["epwing"]))["epwing"];
if(myjson !== undefined)
{
epwing = JSON.parse(myjson);
lookup_epwing_kan = new Map();
lookup_epwing_kana = new Map();
for (let i = 1; i < epwing.length; i++)
{
if(typeof epwing[i] === 'string' || epwing[i] instanceof String)
continue;
add_epwing_reading(epwing[i]["r"], i);
for(let spelling of epwing[i]["s"])
add_epwing_spelling(spelling, i);
}
}
}
refresh_epwing();
let epwing = JSON.parse(myjson);
let name = epwing.shift();
function add_epwing_reading(text, id)
{
if(text === "")
return;
if (!lookup_epwing_kana.has(text))
lookup_epwing_kana.set(text, [id]);
else
lookup_epwing_kana.get(text).push(id);
}
function add_epwing_spelling(text, id)
{
if(text === "")
return;
if (!lookup_epwing_kan.has(text))
lookup_epwing_kan.set(text, [id]);
else
lookup_epwing_kan.get(text).push(id);
let custom_dicts = (await get_storage_or_default("custom_dicts", []));
custom_dicts.push({"name": name, "entries": JSON.stringify(epwing), "enabled": true});
browser.storage.local.set({"custom_dicts":custom_dicts});
browser.storage.local.remove(["epwing"]);
}
refresh_json();
}
migrate_legacy_then_refresh();
let dict = [];
let lookup_kan = new Map();
@@ -758,18 +795,18 @@ function build_lookup_comb(forms)
return;
}
function epwing_lookup_kanji(spelling_list, readings, inexact)
function json_lookup_kanji(dict, spelling_list, readings, inexact)
{
let indexes_set = new Set();
let indexes = [];
for(let spelling of spelling_list)
{
let possibilities = lookup_epwing_kan.get(spelling);
let possibilities = dict.lookup_json_kan.get(spelling);
if(!possibilities)
continue;
for(let id of lookup_epwing_kan.get(spelling))
for(let id of dict.lookup_json_kan.get(spelling))
{
if(inexact || readings.includes(epwing[id]["r"]))
if(inexact || readings.includes(dict.entries[id]["r"]))
{
if(!indexes_set.has(id))
{
@@ -782,36 +819,49 @@ function epwing_lookup_kanji(spelling_list, readings, inexact)
return indexes;
}
function epwing_lookup_kana_exact(kana)
function json_lookup_kana_exact(dict, kana)
{
let possibilities = lookup_epwing_kana.get(kana);
let possibilities = dict.lookup_json_kana.get(dict, kana);
if(!possibilities)
return [];
let actual_possibilities = [];
for(let wing of possibilities)
for(let index of possibilities)
{
let spellings = epwing[wing]["s"]
let spellings = dict.entries[index]["s"]
if(spellings.length == 1 && spellings[0] === "")
actual_possibilities.push(wing);
actual_possibilities.push(index);
}
return actual_possibilities;
}
function epwing_lookup_kana_inexact(kana)
function json_lookup_kana_inexact(dict, kana)
{
let possibilities = copy_gen(lookup_epwing_kana.get(kana));
let possibilities = copy_gen(dict.lookup_json_kana.get(kana));
if(!possibilities)
return [];
return possibilities;
}
function add_epwing_info(lookups, other_settings)
function json_lookup_arbitrary_as_is(dict, text)
{
let index = dict.lookup_json_kan.get(text);
if(index === undefined)
index = dict.lookup_json_kana.get(text);
if(index === undefined)
return undefined;
return copy(dict.entries[index]);
}
function add_json_info(lookups, other_settings)
{
for(let lookup of lookups)
{
for(let entry of lookup.result)
{
// look for an epwing entry that matches this lookup and jmdict entry
let epwing_data = undefined;
entry.json = [];
for(let dict of json_dicts)
{
// look for an json entry that matches this lookup and jmdict entry
let json_data = undefined;
let foundtext = undefined;
if(entry.orig_found.keb)
foundtext = entry.orig_found.keb;
@@ -830,54 +880,54 @@ function add_epwing_info(lookups, other_settings)
readings.push(reading.reb);
// try exact matches
let possibilities = epwing_lookup_kanji(spellings, core_readings_a, false);
let possibilities = json_lookup_kanji(dict, spellings, core_readings_a, false);
if(possibilities.length == 0)
possibilities = epwing_lookup_kanji(spellings, core_readings_b, false);
possibilities = json_lookup_kanji(dict, spellings, core_readings_b, false);
if(possibilities.length == 0)
possibilities = epwing_lookup_kanji(spellings, readings, false);
possibilities = json_lookup_kanji(dict, spellings, readings, false);
// try inexact matches
if(!other_settings.strict_epwing)
{
if(possibilities.length == 0)
possibilities = epwing_lookup_kanji(spellings, core_readings_a, true);
possibilities = json_lookup_kanji(dict, spellings, core_readings_a, true);
if(possibilities.length == 0)
possibilities = epwing_lookup_kanji(spellings, core_readings_b, true);
possibilities = json_lookup_kanji(dict, spellings, core_readings_b, true);
if(possibilities.length == 0)
possibilities = epwing_lookup_kanji(spellings, readings, true);
possibilities = json_lookup_kanji(dict, spellings, readings, true);
}
if(possibilities.length > 0)
epwing_data = copy(epwing[possibilities[0]]);
json_data = copy(dict.entries[possibilities[0]]);
}
else
{
let possibilities = epwing_lookup_kana_exact(foundtext);
let possibilities = json_lookup_kana_exact(foundtext);
if(!other_settings.strict_epwing)
{
for(let reading of entry.r_ele)
{
if(possibilities.length == 0)
possibilities = epwing_lookup_kana_exact(reading.reb);
possibilities = json_lookup_kana_exact(reading.reb);
}
if(possibilities.length == 0)
possibilities = epwing_lookup_kana_inexact(foundtext);
possibilities = json_lookup_kana_inexact(foundtext);
for(let reading of entry.r_ele)
{
if(possibilities.length == 0)
possibilities = epwing_lookup_kana_inexact(reading.reb);
possibilities = json_lookup_kana_inexact(reading.reb);
}
}
if(possibilities.length > 0)
epwing_data = copy(epwing[possibilities[0]]);
json_data = copy(dict.entries[possibilities[0]]);
}
if(epwing_data)
if(json_data)
{
// add dictionary title
if(epwing_data && (typeof epwing[0] === 'string' || epwing[0] instanceof String))
epwing_data["z"] = epwing[0];
json_data["z"] = dict.name;
// add to lookup
entry.epwing = epwing_data;
entry.json.push(json_data);
}
}
}
}
@@ -1256,7 +1306,7 @@ function add_extra_info(results, other_settings)
}
}
}
return add_epwing_info(results, other_settings);
return add_json_info(results, other_settings);
}
// Skip JMdict entries reappearing in alternative lookups (so only the first one is shown)
@@ -1278,7 +1328,7 @@ function skip_rereferenced_entries(results, other_settings)
if(newlookup.length > 0)
newresults.push({text:lookup.text, result:newlookup});
}
return add_extra_info(newresults, other_settings); // add extra information like epwing results and audio data now
return add_extra_info(newresults, other_settings); // add extra information like json results and audio data now
}
let last_lookup = "";
@@ -1494,6 +1544,16 @@ function open_livemining(info, tab)
});
} catch(err) {}
}
function open_jsonconfig(info, tab)
{
try
{
browser.windows.create({
url:browser.extension.getURL("json_config.html"),
type:"popup"
});
} catch(err) {}
}
if(browser.contextMenus)
{
@@ -1515,6 +1575,12 @@ if(browser.contextMenus)
contexts: ["browser_action"],
onclick: open_livemining
});
browser.contextMenus.create({
id: "nazeka-jsonconfig",
title: "Manage JSON Dictionaries",
contexts: ["browser_action"],
onclick: open_jsonconfig
});
}
let origin_tab = undefined;
@@ -1588,9 +1654,9 @@ browser.runtime.onMessage.addListener((req, sender) =>
{
clipboard_hook(sender.tab);
}
else if (req.type == "refreshepwing")
else if (req.type == "refreshjson")
{
refresh_epwing();
refresh_json();
}
else if (req.type == "ankiconnect_mine")
{
+55
View File
@@ -0,0 +1,55 @@
<!doctype html>
<html>
<head>
<meta charset="utf-8"/>
<title>Manage JSON Dictionaries</title>
<style>
body
{
background-color: #111;
color: #CCC;
font-family: Verdana, Arial, IPAGothic, TakaoGothic, Noto Sans CJK JP Regular, Meiryo, sans-serif;
font-size: 1em;
}
option
{
opacity: 0.999;
}
option.disabled
{
opacity: 0.6;
}
div.buttons
{
display:flex;
width: 200px;
flex-direction: row;
flex-wrap: wrap;
justify-content: center;
}
div.buttons *
{
width: 33%;
}
#delete
{
margin-top: 20px;
}
</style>
<script src="include/browser-polyfill.min.js" charset="UTF-8"></script>
<script src="json_config.js" charset="UTF-8"></script>
</head>
<body lang="ja" visible="5">
<select id=dicts size="10" style="width: 200px; box-sizing: border-box; border: none;">
<option value="wadai">Kenkyuusha/Wadai</option>
<option value="shinmeikai">Shinmeikai</option>
<option value="muramasa">Muramasa</option>
</select>
<div class=buttons>
<button id=move_up>Move Up</button><button id=move_down>Move Down</button><button id=add disabled>Add Blank</button>
<button id=import disabled>Import</button><button id=toggle>Toggle</button><button id=edit disabled>Edit</button>
<button id=delete disabled>Delete</button></div>
</body>
</html>
+102
View File
@@ -0,0 +1,102 @@
let dicts = [];
function save_dicts()
{
browser.storage.local.set({"custom_dicts": dicts});
browser.runtime.sendMessage({type:"refreshjson"});
}
function move_up()
{
let list = document.getElementById("dicts");
let index = list.selectedIndex;
if(index == -1 || index == 0)
return;
list.add(list[index], list[index-1]);
let temp = dicts[index];
dicts[index] = dicts[index-1];
dicts[index-1] = temp;
save_dicts();
}
function move_down()
{
let list = document.getElementById("dicts");
let index = list.selectedIndex;
if(index == -1 || index == list.length-1)
return;
list.add(list[index], list[index+2]);
let temp = dicts[index];
dicts[index] = dicts[index+1];
dicts[index+1] = temp;
save_dicts();
}
function toggle()
{
let list = document.getElementById("dicts");
let index = list.selectedIndex;
if(index == -1)
return;
if(list[index].className == "disabled")
{
list[index].className = "";
}
else
{
list[index].className = "disabled";
}
dicts[index].enabled = !dicts[index].enabled;
save_dicts();
}
function or_default(val, fallback)
{
if(val !== undefined)
return val;
else
return fallback;
}
async function get_storage_or_default(name, fallback)
{
return or_default((await browser.storage.local.get(name))[name], fallback);
}
async function insert_dicts()
{
let list = document.getElementById("dicts");
list.innerHTML = "";
dicts = (await get_storage_or_default("custom_dicts", []));
for(let stored_dict of dicts)
{
let option = document.createElement("option");
option.innerText = stored_dict.name;
option.value = stored_dict.name;
if(!stored_dict.enabled)
option.className = "disabled";
list.appendChild(option);
}
}
function buildpage()
{
document.getElementById("move_up").onclick = move_up;
document.getElementById("move_down").onclick = move_down;
document.getElementById("toggle").onclick = toggle;
insert_dicts();
}
if (document.readyState == "complete")
{
buildpage();
}
else
{
document.addEventListener("DOMContentLoaded", buildpage);
}
+2
View File
@@ -49,6 +49,8 @@
"mining.html",
"livemining.js",
"livemining.html",
"json_config.js",
"json_config.html",
"jmdictabbreviations.html"
],
+1
View File
@@ -1,3 +1,4 @@
<!doctype html>
<html>
<head>
<meta charset="utf-8"/>
+1 -97
View File
@@ -627,102 +627,6 @@ function buildpage()
optionsection.appendChild(document.createElement("hr"));
// json dictionary
let file = document.createElement("input");
file.type = "file";
file.id = "epwing_file";
file.addEventListener("change", () =>
{
let fname = document.querySelector("#epwing_file").files[0];
let reader = new FileReader();
reader.onload = async(e) =>
{
// FIXME: add asking for the unlimited storage permission
try
{
browser.storage.local.set({"epwing":JSON.stringify(JSON.parse(e.target.result))});
browser.runtime.sendMessage({type:"refreshepwing"});
document.querySelector("#import_label").textContent = "Imported. Might take a few seconds to apply.";
}
catch(except)
{
console.log(except.stack);
}
};
reader.readAsText(fname);
});
let label = document.createElement("label");
label.for = file.id;
label.id = "import_label";
label.textContent = "Import JSON dictionary.";
label.style.display = "block";
function json_dict_reset()
{
browser.storage.local.set({"epwing":JSON.stringify(JSON.parse("[]"))});
browser.runtime.sendMessage({type:"refreshepwing"});
document.body.querySelector("#epwing_reset").innerText = "Delete JSON dictionary";
document.body.querySelector("#epwing_reset").addEventListener("click", json_dict_ask_reset, {once : true});
}
function json_dict_ask_reset()
{
document.body.querySelector("#epwing_reset").innerText = "Click again to make Nazeka to delete the JSON dictionary.";
document.body.querySelector("#epwing_reset").addEventListener("click", json_dict_reset, {once : true});
}
let jsonresetbutton = document.createElement("button");
jsonresetbutton.type = "button";
jsonresetbutton.id = "epwing_reset";
jsonresetbutton.innerText = "Delete JSON dictionary";
jsonresetbutton.addEventListener("click", json_dict_ask_reset, {once : true});
let dict_explanation = document.createElement("p");
dict_explanation.innerHTML =
`JSON dictionaries have the following format:
<pre style='font-family:monospace !important' lang='en-US'>[
"The Worst Dictionary Ever",
{
"r":"あ",
"s":[
"亜",
"亞"
],
"l":[
"this is a single line of a definition"
]
},
{
"r":"ああ",
"s":[
""
],
"l":[
"1) yep",
"  1 - this is a multi-line, multi-part definition",
"  2 - dictionaries are crazy",
"2) why is this word in a dictionary?"
]
},
//...
]</pre>
JSON dictionaries are JSON files consisting of a single array. The first entry is a string, the name of the dictionary. All remaining entries are objects, each containing the three keys "r", "s", and "l".<br>
"r" maps to a single string, the reading of that entry.<br>
"s" maps to an array listing strings that are all the possible spellings of the current entry. If the only spelling is the reading, there is a single blank string in the array of spellings.<br>
"l" maps to an array of strings, each of which are one of the lines of the definition, in order.<br>
This format is designed to be a sane generic mapping for EPWing dictionaries, after all the metadata is stripped out, and all "gaiji" encodings have been converted to unicode already.<br>
You can currently only have one JSON dictionary at a time and cannot manage the one you have imported aside from replacing it.<br>
Unfortunately, EPWING dictionaries can't be converted to this format in a general way. Someone has to write a script for every single EPWING dictionary out there, one at a time.<br>
This format can be extended in the future if it becomes desirable to include more information, like pitch accent data, from dictionaries that have it.`;
optionsection.appendChild(file);
optionsection.appendChild(label);
optionsection.appendChild(jsonresetbutton);
optionsection.appendChild(dict_explanation);
optionsection.appendChild(document.createElement("hr"));
// backup
let backup_save = document.createElement("button");
@@ -757,7 +661,7 @@ This format can be extended in the future if it becomes desirable to include mor
try
{
browser.storage.local.set(JSON.parse(e.target.result));
browser.runtime.sendMessage({type:"refreshepwing"});
browser.runtime.sendMessage({type:"refreshjson"});
document.querySelector("#backup_load_label").textContent = "Loaded backup. Might take a few seconds to apply.";
}
catch(except)
+1
View File
@@ -1,3 +1,4 @@
<!doctype html>
<html>
<head>
<meta charset="utf-8"/>
+38 -36
View File
@@ -507,48 +507,50 @@ function elementize_jmdict_defs(goodsenses)
}
return jmdict_defs;
}
function elementize_epwing_defs(epwing)
function elementize_json_defs(json)
{
if(epwing)
if(json.length > 0)
{
let epwing_defs = document.createElement("div");
epwing_defs.className = "epwing_definitions";
let json_defs = document.createElement("div");
json_defs.className = "json_definitions";
let epwing_head = document.createElement("div");
epwing_head.appendChild(document.createTextNode("―"+epwing["z"]+"―"));
epwing_head.appendChild(document.createElement("br"));
let epwing_head_text = epwing["r"];
if(epwing["s"] && epwing["s"][0] != "")
for(let entry of json)
{
let json_head = document.createElement("div");
json_head.appendChild(document.createTextNode("―"+entry["z"]+"―"));
json_head.appendChild(document.createElement("br"));
let json_head_text = entry["r"];
if(entry["s"] && entry["s"][0] != "")
{
let isfirst = true;
epwing_head_text += "【";
for(let spelling of epwing["s"])
json_head_text += "【";
for(let spelling of entry["s"])
{
if(!isfirst)
epwing_head_text += "・";
epwing_head_text += spelling;
json_head_text += "・";
json_head_text += spelling;
isfirst = false;
}
epwing_head_text += "】";
json_head_text += "】";
}
epwing_head.appendChild(document.createTextNode(epwing_head_text));
let epwing_definition = document.createElement("div");
json_head.appendChild(document.createTextNode(json_head_text));
let json_definition = document.createElement("div");
let isfirst = true;
for(let line of epwing["l"])
for(let line of entry["l"])
{
if(!isfirst)
epwing_definition.appendChild(document.createElement("br"));
json_definition.appendChild(document.createElement("br"));
let def_line = document.createElement("span");
def_line.textContent = line;
epwing_definition.appendChild(def_line);
json_definition.appendChild(def_line);
isfirst = false;
}
epwing_head.className = "epwing_head";
epwing_definition.className = "epwing_definition";
epwing_defs.appendChild(epwing_head);
epwing_defs.appendChild(epwing_definition);
return epwing_defs;
json_head.className = "json_head";
json_definition.className = "json_definition";
json_defs.appendChild(json_head);
json_defs.appendChild(json_definition);
}
return json_defs;
}
else
{
@@ -574,9 +576,9 @@ function get_style()
.nazeka_sub_keb{font-size:${settings.reading_fontsize}px;white-space:nowrap;font-family: ${font}IPAGothic,TakaoGothic,Noto Sans CJK JP Regular,Meiryo,sans-serif}
.nazeka_sub_reb{font-size:${settings.reading_fontsize}px;white-space:nowrap;color:${settings.hlcolor2}}
.nazeka_original{float: right; margin-right: 2px; margin-left:4px; opacity:0.7;}
.epwing_head{margin: 0 4px 6px;}
.json_head{margin: 0 4px 6px;}
.jmdict_definitions{margin: 0 4px 6px;}
.epwing_definition{margin: 0 4px 6px;}
.json_definition{margin: 0 4px 6px;}
.kanji_info{margin: 2px 4px 2px;}
`;
return style;
@@ -984,30 +986,30 @@ function build_div_inner(text, result, moreText, index, first_of_many = false)
definition.className = "nazeka_definitions";
let jmdict_div = elementize_jmdict_defs(term.sense);
let epwing_div = elementize_epwing_defs(term.epwing);
let json_div = elementize_json_defs(term.json);
if (settings.definitions_mode == 0 || (settings.normal_definitions_in_mining && mining_ui_exists())) // normal
{
if(jmdict_div) definition.appendChild(jmdict_div);
if(epwing_div) definition.appendChild(epwing_div);
if(json_div) definition.appendChild(json_div);
}
else if (settings.definitions_mode == 1) // epwing first
else if (settings.definitions_mode == 1) // json first
{
if(epwing_div) definition.appendChild(epwing_div);
if(json_div) definition.appendChild(json_div);
if(jmdict_div)
{
jmdict_div.insertBefore(document.createTextNode("jmdict: "), jmdict_div.firstChild);
definition.appendChild(jmdict_div);
}
}
else if (settings.definitions_mode == 2) // epwing or else jmdict
else if (settings.definitions_mode == 2) // json or else jmdict
{
if(epwing_div) definition.appendChild(epwing_div);
if(json_div) definition.appendChild(json_div);
else definition.appendChild(jmdict_div);
}
else if (settings.definitions_mode == 3) // epwing only
else if (settings.definitions_mode == 3) // json only
{
if(epwing_div) definition.appendChild(epwing_div);
if(json_div) definition.appendChild(json_div);
}
else if (settings.definitions_mode == 4) // none
{
@@ -1016,7 +1018,7 @@ function build_div_inner(text, result, moreText, index, first_of_many = false)
else // normal (fallback)
{
if(jmdict_div) definition.appendChild(jmdict_div);
if(epwing_div) definition.appendChild(epwing_div);
if(json_div) definition.appendChild(json_div);
}
container.appendChild(definition);