From 2a69981d6e04307bfb5cefa59b5c28bfdcd5629f Mon Sep 17 00:00:00 2001 From: wareya Date: Mon, 13 May 2019 20:01:11 -0400 Subject: [PATCH] basic epwing/json dictionary migration and management done --- background-script.js | 282 ++++++++++++++++++++++++++----------------- json_config.html | 55 +++++++++ json_config.js | 102 ++++++++++++++++ manifest.json | 2 + mining.html | 1 + options.js | 98 +-------------- reader.html | 1 + texthook.js | 88 +++++++------- 8 files changed, 381 insertions(+), 248 deletions(-) create mode 100644 json_config.html create mode 100644 json_config.js diff --git a/background-script.js b/background-script.js index b398059..161e623 100644 --- a/background-script.js +++ b/background-script.js @@ -48,49 +48,86 @@ browser.storage.onChanged.addListener((updates, storageArea) => // We have to send an HTTP request and store the result in a string before parsing it into an object. // Seriously. -let epwing = []; -let lookup_epwing_kan = new Map(); -let lookup_epwing_kana = new Map(); +let json_dicts = []; -async function refresh_epwing() +function add_json_reading(dict, text, id) { - let myjson = (await browser.storage.local.get("epwing"))["epwing"]; - if(myjson !== undefined) + if(text === "") + return; + if (!dict.lookup_json_kana.has(text)) + dict.lookup_json_kana.set(text, [id]); + else + dict.lookup_json_kana.get(text).push(id); +} +function add_json_spelling(dict, text, id) +{ + if(text === "") + return; + if (!dict.lookup_json_kan.has(text)) + dict.lookup_json_kan.set(text, [id]); + else + dict.lookup_json_kan.get(text).push(id); +} + +function or_default(val, fallback) +{ + if(val !== undefined) + return val; + else + return fallback; +} +async function get_storage_or_default(name, fallback) +{ + return or_default((await browser.storage.local.get(name))[name], fallback); +} + +async function refresh_json() +{ + let custom_dicts = (await get_storage_or_default("custom_dicts", [])); + json_dicts = []; + for(let stored_dict of custom_dicts) { - epwing = JSON.parse(myjson); - lookup_epwing_kan = new Map(); - lookup_epwing_kana = new Map(); - for (let i = 1; i < epwing.length; i++) + if(!stored_dict.enabled) + continue; + let name = stored_dict.name; + let entries = JSON.parse(stored_dict.entries); + let lookup_json_kan = new Map(); + let lookup_json_kana = new Map(); + let dict = {"name": name, "entries": entries, "lookup_json_kan": lookup_json_kan, "lookup_json_kana": lookup_json_kana}; + try { - if(typeof epwing[i] === 'string' || epwing[i] instanceof String) - continue; - add_epwing_reading(epwing[i]["r"], i); - for(let spelling of epwing[i]["s"]) - add_epwing_spelling(spelling, i); - } + for (let i = 0; i < entries.length; i++) + { + add_json_reading(dict, entries[i]["r"], i); + for(let spelling of entries[i]["s"]) + add_json_spelling(dict, spelling, i); + } + } catch(e){} + json_dicts.push(dict); } } -refresh_epwing(); -function add_epwing_reading(text, id) +async function migrate_legacy_then_refresh() { - if(text === "") - return; - if (!lookup_epwing_kana.has(text)) - lookup_epwing_kana.set(text, [id]); - else - lookup_epwing_kana.get(text).push(id); -} -function add_epwing_spelling(text, id) -{ - if(text === "") - return; - if (!lookup_epwing_kan.has(text)) - lookup_epwing_kan.set(text, [id]); - else - lookup_epwing_kan.get(text).push(id); + let myjson = (await browser.storage.local.get(["epwing"]))["epwing"]; + if(myjson !== undefined) + { + let epwing = JSON.parse(myjson); + let name = epwing.shift(); + + let custom_dicts = (await get_storage_or_default("custom_dicts", [])); + custom_dicts.push({"name": name, "entries": JSON.stringify(epwing), "enabled": true}); + + browser.storage.local.set({"custom_dicts":custom_dicts}); + + browser.storage.local.remove(["epwing"]); + } + + refresh_json(); } +migrate_legacy_then_refresh(); + let dict = []; let lookup_kan = new Map(); @@ -758,18 +795,18 @@ function build_lookup_comb(forms) return; } -function epwing_lookup_kanji(spelling_list, readings, inexact) +function json_lookup_kanji(dict, spelling_list, readings, inexact) { let indexes_set = new Set(); let indexes = []; for(let spelling of spelling_list) { - let possibilities = lookup_epwing_kan.get(spelling); + let possibilities = dict.lookup_json_kan.get(spelling); if(!possibilities) continue; - for(let id of lookup_epwing_kan.get(spelling)) + for(let id of dict.lookup_json_kan.get(spelling)) { - if(inexact || readings.includes(epwing[id]["r"])) + if(inexact || readings.includes(dict.entries[id]["r"])) { if(!indexes_set.has(id)) { @@ -782,102 +819,115 @@ function epwing_lookup_kanji(spelling_list, readings, inexact) return indexes; } -function epwing_lookup_kana_exact(kana) +function json_lookup_kana_exact(dict, kana) { - let possibilities = lookup_epwing_kana.get(kana); + let possibilities = dict.lookup_json_kana.get(dict, kana); if(!possibilities) return []; let actual_possibilities = []; - for(let wing of possibilities) + for(let index of possibilities) { - let spellings = epwing[wing]["s"] + let spellings = dict.entries[index]["s"] if(spellings.length == 1 && spellings[0] === "") - actual_possibilities.push(wing); + actual_possibilities.push(index); } return actual_possibilities; } -function epwing_lookup_kana_inexact(kana) +function json_lookup_kana_inexact(dict, kana) { - let possibilities = copy_gen(lookup_epwing_kana.get(kana)); + let possibilities = copy_gen(dict.lookup_json_kana.get(kana)); if(!possibilities) return []; return possibilities; } -function add_epwing_info(lookups, other_settings) +function json_lookup_arbitrary_as_is(dict, text) +{ + let index = dict.lookup_json_kan.get(text); + if(index === undefined) + index = dict.lookup_json_kana.get(text); + if(index === undefined) + return undefined; + return copy(dict.entries[index]); +} + +function add_json_info(lookups, other_settings) { for(let lookup of lookups) { for(let entry of lookup.result) { - // look for an epwing entry that matches this lookup and jmdict entry - let epwing_data = undefined; - let foundtext = undefined; - if(entry.orig_found.keb) - foundtext = entry.orig_found.keb; - else - foundtext = entry.orig_found.reb; - - if(entry.k_ele) + entry.json = []; + for(let dict of json_dicts) { - let spellings = [lookup.text, foundtext]; - for(let spelling of entry.k_ele) - spellings.push(spelling.keb); - let core_readings_a = [lookup.text, foundtext]; - let core_readings_b = [entry.r_ele[0].reb]; - let readings = []; - for(let reading of entry.r_ele) - readings.push(reading.reb); + // look for an json entry that matches this lookup and jmdict entry + let json_data = undefined; + let foundtext = undefined; + if(entry.orig_found.keb) + foundtext = entry.orig_found.keb; + else + foundtext = entry.orig_found.reb; - // try exact matches - let possibilities = epwing_lookup_kanji(spellings, core_readings_a, false); - if(possibilities.length == 0) - possibilities = epwing_lookup_kanji(spellings, core_readings_b, false); - if(possibilities.length == 0) - possibilities = epwing_lookup_kanji(spellings, readings, false); - // try inexact matches - if(!other_settings.strict_epwing) - { - if(possibilities.length == 0) - possibilities = epwing_lookup_kanji(spellings, core_readings_a, true); - if(possibilities.length == 0) - possibilities = epwing_lookup_kanji(spellings, core_readings_b, true); - if(possibilities.length == 0) - possibilities = epwing_lookup_kanji(spellings, readings, true); - } - - if(possibilities.length > 0) - epwing_data = copy(epwing[possibilities[0]]); - } - else - { - let possibilities = epwing_lookup_kana_exact(foundtext); - if(!other_settings.strict_epwing) + if(entry.k_ele) { + let spellings = [lookup.text, foundtext]; + for(let spelling of entry.k_ele) + spellings.push(spelling.keb); + let core_readings_a = [lookup.text, foundtext]; + let core_readings_b = [entry.r_ele[0].reb]; + let readings = []; for(let reading of entry.r_ele) + readings.push(reading.reb); + + // try exact matches + let possibilities = json_lookup_kanji(dict, spellings, core_readings_a, false); + if(possibilities.length == 0) + possibilities = json_lookup_kanji(dict, spellings, core_readings_b, false); + if(possibilities.length == 0) + possibilities = json_lookup_kanji(dict, spellings, readings, false); + // try inexact matches + if(!other_settings.strict_epwing) { if(possibilities.length == 0) - possibilities = epwing_lookup_kana_exact(reading.reb); - } - if(possibilities.length == 0) - possibilities = epwing_lookup_kana_inexact(foundtext); - for(let reading of entry.r_ele) - { + possibilities = json_lookup_kanji(dict, spellings, core_readings_a, true); if(possibilities.length == 0) - possibilities = epwing_lookup_kana_inexact(reading.reb); + possibilities = json_lookup_kanji(dict, spellings, core_readings_b, true); + if(possibilities.length == 0) + possibilities = json_lookup_kanji(dict, spellings, readings, true); } + + if(possibilities.length > 0) + json_data = copy(dict.entries[possibilities[0]]); + } + else + { + let possibilities = json_lookup_kana_exact(foundtext); + if(!other_settings.strict_epwing) + { + for(let reading of entry.r_ele) + { + if(possibilities.length == 0) + possibilities = json_lookup_kana_exact(reading.reb); + } + if(possibilities.length == 0) + possibilities = json_lookup_kana_inexact(foundtext); + for(let reading of entry.r_ele) + { + if(possibilities.length == 0) + possibilities = json_lookup_kana_inexact(reading.reb); + } + } + if(possibilities.length > 0) + json_data = copy(dict.entries[possibilities[0]]); + } + + if(json_data) + { + // add dictionary title + json_data["z"] = dict.name; + // add to lookup + entry.json.push(json_data); } - if(possibilities.length > 0) - epwing_data = copy(epwing[possibilities[0]]); - } - - if(epwing_data) - { - // add dictionary title - if(epwing_data && (typeof epwing[0] === 'string' || epwing[0] instanceof String)) - epwing_data["z"] = epwing[0]; - // add to lookup - entry.epwing = epwing_data; } } } @@ -1256,7 +1306,7 @@ function add_extra_info(results, other_settings) } } } - return add_epwing_info(results, other_settings); + return add_json_info(results, other_settings); } // Skip JMdict entries reappearing in alternative lookups (so only the first one is shown) @@ -1278,7 +1328,7 @@ function skip_rereferenced_entries(results, other_settings) if(newlookup.length > 0) newresults.push({text:lookup.text, result:newlookup}); } - return add_extra_info(newresults, other_settings); // add extra information like epwing results and audio data now + return add_extra_info(newresults, other_settings); // add extra information like json results and audio data now } let last_lookup = ""; @@ -1494,6 +1544,16 @@ function open_livemining(info, tab) }); } catch(err) {} } +function open_jsonconfig(info, tab) +{ + try + { + browser.windows.create({ + url:browser.extension.getURL("json_config.html"), + type:"popup" + }); + } catch(err) {} +} if(browser.contextMenus) { @@ -1515,6 +1575,12 @@ if(browser.contextMenus) contexts: ["browser_action"], onclick: open_livemining }); + browser.contextMenus.create({ + id: "nazeka-jsonconfig", + title: "Manage JSON Dictionaries", + contexts: ["browser_action"], + onclick: open_jsonconfig + }); } let origin_tab = undefined; @@ -1588,9 +1654,9 @@ browser.runtime.onMessage.addListener((req, sender) => { clipboard_hook(sender.tab); } - else if (req.type == "refreshepwing") + else if (req.type == "refreshjson") { - refresh_epwing(); + refresh_json(); } else if (req.type == "ankiconnect_mine") { diff --git a/json_config.html b/json_config.html new file mode 100644 index 0000000..1e5ea6e --- /dev/null +++ b/json_config.html @@ -0,0 +1,55 @@ + + + + +Manage JSON Dictionaries + + + + + + + +
+ + +
+ + + diff --git a/json_config.js b/json_config.js new file mode 100644 index 0000000..30b86c6 --- /dev/null +++ b/json_config.js @@ -0,0 +1,102 @@ +let dicts = []; + +function save_dicts() +{ + browser.storage.local.set({"custom_dicts": dicts}); + browser.runtime.sendMessage({type:"refreshjson"}); +} + +function move_up() +{ + let list = document.getElementById("dicts"); + let index = list.selectedIndex; + if(index == -1 || index == 0) + return; + list.add(list[index], list[index-1]); + + let temp = dicts[index]; + dicts[index] = dicts[index-1]; + dicts[index-1] = temp; + + save_dicts(); +} +function move_down() +{ + let list = document.getElementById("dicts"); + let index = list.selectedIndex; + if(index == -1 || index == list.length-1) + return; + list.add(list[index], list[index+2]); + + let temp = dicts[index]; + dicts[index] = dicts[index+1]; + dicts[index+1] = temp; + + save_dicts(); +} +function toggle() +{ + let list = document.getElementById("dicts"); + let index = list.selectedIndex; + if(index == -1) + return; + if(list[index].className == "disabled") + { + list[index].className = ""; + } + else + { + list[index].className = "disabled"; + } + dicts[index].enabled = !dicts[index].enabled; + + save_dicts(); +} + +function or_default(val, fallback) +{ + if(val !== undefined) + return val; + else + return fallback; +} +async function get_storage_or_default(name, fallback) +{ + return or_default((await browser.storage.local.get(name))[name], fallback); +} + +async function insert_dicts() +{ + let list = document.getElementById("dicts"); + list.innerHTML = ""; + + dicts = (await get_storage_or_default("custom_dicts", [])); + + for(let stored_dict of dicts) + { + let option = document.createElement("option"); + option.innerText = stored_dict.name; + option.value = stored_dict.name; + if(!stored_dict.enabled) + option.className = "disabled"; + list.appendChild(option); + } +} + +function buildpage() +{ + document.getElementById("move_up").onclick = move_up; + document.getElementById("move_down").onclick = move_down; + document.getElementById("toggle").onclick = toggle; + + insert_dicts(); +} + +if (document.readyState == "complete") +{ + buildpage(); +} +else +{ + document.addEventListener("DOMContentLoaded", buildpage); +} diff --git a/manifest.json b/manifest.json index cb0fadc..9ad1f29 100644 --- a/manifest.json +++ b/manifest.json @@ -49,6 +49,8 @@ "mining.html", "livemining.js", "livemining.html", + "json_config.js", + "json_config.html", "jmdictabbreviations.html" ], diff --git a/mining.html b/mining.html index 2d1540b..08b2255 100644 --- a/mining.html +++ b/mining.html @@ -1,3 +1,4 @@ + diff --git a/options.js b/options.js index 0be5433..708c8f1 100644 --- a/options.js +++ b/options.js @@ -627,102 +627,6 @@ function buildpage() optionsection.appendChild(document.createElement("hr")); - // json dictionary - - let file = document.createElement("input"); - file.type = "file"; - file.id = "epwing_file"; - file.addEventListener("change", () => - { - let fname = document.querySelector("#epwing_file").files[0]; - let reader = new FileReader(); - reader.onload = async(e) => - { - // FIXME: add asking for the unlimited storage permission - try - { - browser.storage.local.set({"epwing":JSON.stringify(JSON.parse(e.target.result))}); - browser.runtime.sendMessage({type:"refreshepwing"}); - document.querySelector("#import_label").textContent = "Imported. Might take a few seconds to apply."; - } - catch(except) - { - console.log(except.stack); - } - }; - reader.readAsText(fname); - }); - let label = document.createElement("label"); - label.for = file.id; - label.id = "import_label"; - label.textContent = "Import JSON dictionary."; - label.style.display = "block"; - - function json_dict_reset() - { - browser.storage.local.set({"epwing":JSON.stringify(JSON.parse("[]"))}); - browser.runtime.sendMessage({type:"refreshepwing"}); - document.body.querySelector("#epwing_reset").innerText = "Delete JSON dictionary"; - document.body.querySelector("#epwing_reset").addEventListener("click", json_dict_ask_reset, {once : true}); - } - - function json_dict_ask_reset() - { - document.body.querySelector("#epwing_reset").innerText = "Click again to make Nazeka to delete the JSON dictionary."; - document.body.querySelector("#epwing_reset").addEventListener("click", json_dict_reset, {once : true}); - } - - let jsonresetbutton = document.createElement("button"); - jsonresetbutton.type = "button"; - jsonresetbutton.id = "epwing_reset"; - jsonresetbutton.innerText = "Delete JSON dictionary"; - jsonresetbutton.addEventListener("click", json_dict_ask_reset, {once : true}); - - let dict_explanation = document.createElement("p"); - dict_explanation.innerHTML = -`JSON dictionaries have the following format: -
[
-    "The Worst Dictionary Ever",
-    {
-        "r":"あ",
-        "s":[
-            "亜",
-            "亞"
-        ],
-        "l":[
-            "this is a single line of a definition"
-        ]
-    },
-    {
-        "r":"ああ",
-        "s":[
-            ""
-        ],
-        "l":[
-            "1) yep",
-            "　　1 - this is a multi-line, multi-part definition",
-            "　　2 - dictionaries are crazy",
-            "2) why is this word in a dictionary?"
-        ]
-    },
-    //...
-]
-JSON dictionaries are JSON files consisting of a single array. The first entry is a string, the name of the dictionary. All remaining entries are objects, each containing the three keys "r", "s", and "l".
-"r" maps to a single string, the reading of that entry.
-"s" maps to an array listing strings that are all the possible spellings of the current entry. If the only spelling is the reading, there is a single blank string in the array of spellings.
-"l" maps to an array of strings, each of which are one of the lines of the definition, in order.
-This format is designed to be a sane generic mapping for EPWing dictionaries, after all the metadata is stripped out, and all "gaiji" encodings have been converted to unicode already.
-You can currently only have one JSON dictionary at a time and cannot manage the one you have imported aside from replacing it.
-Unfortunately, EPWING dictionaries can't be converted to this format in a general way. Someone has to write a script for every single EPWING dictionary out there, one at a time.
-This format can be extended in the future if it becomes desirable to include more information, like pitch accent data, from dictionaries that have it.`; - - optionsection.appendChild(file); - optionsection.appendChild(label); - optionsection.appendChild(jsonresetbutton); - optionsection.appendChild(dict_explanation); - - optionsection.appendChild(document.createElement("hr")); - // backup let backup_save = document.createElement("button"); @@ -757,7 +661,7 @@ This format can be extended in the future if it becomes desirable to include mor try { browser.storage.local.set(JSON.parse(e.target.result)); - browser.runtime.sendMessage({type:"refreshepwing"}); + browser.runtime.sendMessage({type:"refreshjson"}); document.querySelector("#backup_load_label").textContent = "Loaded backup. Might take a few seconds to apply."; } catch(except) diff --git a/reader.html b/reader.html index 82d9746..58ee569 100644 --- a/reader.html +++ b/reader.html @@ -1,3 +1,4 @@ + diff --git a/texthook.js b/texthook.js index 2026477..fb13774 100644 --- a/texthook.js +++ b/texthook.js @@ -507,48 +507,50 @@ function elementize_jmdict_defs(goodsenses) } return jmdict_defs; } -function elementize_epwing_defs(epwing) +function elementize_json_defs(json) { - if(epwing) + if(json.length > 0) { - let epwing_defs = document.createElement("div"); - epwing_defs.className = "epwing_definitions"; + let json_defs = document.createElement("div"); + json_defs.className = "json_definitions"; - let epwing_head = document.createElement("div"); - epwing_head.appendChild(document.createTextNode("―"+epwing["z"]+"―")); - epwing_head.appendChild(document.createElement("br")); - let epwing_head_text = epwing["r"]; - if(epwing["s"] && epwing["s"][0] != "") + for(let entry of json) { + let json_head = document.createElement("div"); + json_head.appendChild(document.createTextNode("―"+entry["z"]+"―")); + json_head.appendChild(document.createElement("br")); + let json_head_text = entry["r"]; + if(entry["s"] && entry["s"][0] != "") + { + let isfirst = true; + json_head_text += "【"; + for(let spelling of entry["s"]) + { + if(!isfirst) + json_head_text += "・"; + json_head_text += spelling; + isfirst = false; + } + json_head_text += "】"; + } + json_head.appendChild(document.createTextNode(json_head_text)); + let json_definition = document.createElement("div"); let isfirst = true; - epwing_head_text += "【"; - for(let spelling of epwing["s"]) + for(let line of entry["l"]) { if(!isfirst) - epwing_head_text += "・"; - epwing_head_text += spelling; + json_definition.appendChild(document.createElement("br")); + let def_line = document.createElement("span"); + def_line.textContent = line; + json_definition.appendChild(def_line); isfirst = false; } - epwing_head_text += "】"; + json_head.className = "json_head"; + json_definition.className = "json_definition"; + json_defs.appendChild(json_head); + json_defs.appendChild(json_definition); } - epwing_head.appendChild(document.createTextNode(epwing_head_text)); - let epwing_definition = document.createElement("div"); - let isfirst = true; - for(let line of epwing["l"]) - { - if(!isfirst) - epwing_definition.appendChild(document.createElement("br")); - let def_line = document.createElement("span"); - def_line.textContent = line; - epwing_definition.appendChild(def_line); - isfirst = false; - } - epwing_head.className = "epwing_head"; - epwing_definition.className = "epwing_definition"; - epwing_defs.appendChild(epwing_head); - epwing_defs.appendChild(epwing_definition); - - return epwing_defs; + return json_defs; } else { @@ -574,9 +576,9 @@ function get_style() .nazeka_sub_keb{font-size:${settings.reading_fontsize}px;white-space:nowrap;font-family: ${font}IPAGothic,TakaoGothic,Noto Sans CJK JP Regular,Meiryo,sans-serif} .nazeka_sub_reb{font-size:${settings.reading_fontsize}px;white-space:nowrap;color:${settings.hlcolor2}} .nazeka_original{float: right; margin-right: 2px; margin-left:4px; opacity:0.7;} -.epwing_head{margin: 0 4px 6px;} +.json_head{margin: 0 4px 6px;} .jmdict_definitions{margin: 0 4px 6px;} -.epwing_definition{margin: 0 4px 6px;} +.json_definition{margin: 0 4px 6px;} .kanji_info{margin: 2px 4px 2px;} `; return style; @@ -984,30 +986,30 @@ function build_div_inner(text, result, moreText, index, first_of_many = false) definition.className = "nazeka_definitions"; let jmdict_div = elementize_jmdict_defs(term.sense); - let epwing_div = elementize_epwing_defs(term.epwing); + let json_div = elementize_json_defs(term.json); if (settings.definitions_mode == 0 || (settings.normal_definitions_in_mining && mining_ui_exists())) // normal { if(jmdict_div) definition.appendChild(jmdict_div); - if(epwing_div) definition.appendChild(epwing_div); + if(json_div) definition.appendChild(json_div); } - else if (settings.definitions_mode == 1) // epwing first + else if (settings.definitions_mode == 1) // json first { - if(epwing_div) definition.appendChild(epwing_div); + if(json_div) definition.appendChild(json_div); if(jmdict_div) { jmdict_div.insertBefore(document.createTextNode("jmdict: "), jmdict_div.firstChild); definition.appendChild(jmdict_div); } } - else if (settings.definitions_mode == 2) // epwing or else jmdict + else if (settings.definitions_mode == 2) // json or else jmdict { - if(epwing_div) definition.appendChild(epwing_div); + if(json_div) definition.appendChild(json_div); else definition.appendChild(jmdict_div); } - else if (settings.definitions_mode == 3) // epwing only + else if (settings.definitions_mode == 3) // json only { - if(epwing_div) definition.appendChild(epwing_div); + if(json_div) definition.appendChild(json_div); } else if (settings.definitions_mode == 4) // none { @@ -1016,7 +1018,7 @@ function build_div_inner(text, result, moreText, index, first_of_many = false) else // normal (fallback) { if(jmdict_div) definition.appendChild(jmdict_div); - if(epwing_div) definition.appendChild(epwing_div); + if(json_div) definition.appendChild(json_div); } container.appendChild(definition);