finally, frequency data

background-script.js

@@ -9,6 +9,7 @@ let settings = {
 reader_width: 800,
 reader_height: 300,
 deconjugator_rules_json: "",
+freqlist_mode: 1,
 };
 
 async function settings_init()
@@ -25,6 +26,7 @@ async function settings_init()
         getvar("reader_width", 800);
         getvar("reader_height", 300);
         getvar("deconjugator_rules_json", "");
+        getvar("freqlist_mode", 1);
     } catch(err) {} // options not stored yet
 }
 
@@ -480,6 +482,52 @@ function load_priority_rules(filename)
 
 load_priority_rules("dict/priority.json");
 
+// vn frequency data
+let freq_vns = {};
+
+function load_freqlist_vns(filename)
+{
+    let req = new XMLHttpRequest();
+    req.addEventListener("load", () =>
+    {
+        if (req.readyState === 4)
+        {
+            console.log("loaded vn freq data");
+            if (req.status === 200)
+                freq_vns = JSON.parse(req.responseText);
+            else
+                console.error(req.statusText);
+        }
+    });
+    req.open("GET", browser.extension.getURL(filename));
+    req.send();
+    return req;
+}
+load_freqlist_vns("dict/freqlist_vns.json");
+
+// narou frequency data
+let freq_narou = {};
+
+function load_freqlist_narou(filename)
+{
+    let req = new XMLHttpRequest();
+    req.addEventListener("load", () =>
+    {
+        if (req.readyState === 4)
+        {
+            console.log("loaded narou freq data");
+            if (req.status === 200)
+                freq_narou = JSON.parse(req.responseText);
+            else
+                console.error(req.statusText);
+        }
+    });
+    req.open("GET", browser.extension.getURL(filename));
+    req.send();
+    return req;
+}
+load_freqlist_narou("dict/freqlist_narou.json");
+
 // deconjugation rules
 let rules = [];
 
@@ -1466,7 +1514,52 @@ function add_extra_info(results, other_settings)
             }
         }
     }
-    return add_json_info(results, other_settings);
+    let ret = add_json_info(results, other_settings);
+    
+    if(settings.freqlist_mode > 0)
+    {
+        for(let lookup of results)
+        {
+            for(let entry of lookup.result)
+            {
+                let seen = new Set();
+                for(let r of entry.r_ele)
+                    seen.add(r.reb);
+                if(entry.k_ele)
+                    for(let k of entry.k_ele)
+                        seen.add(k.keb);
+                entry.freq = [];
+                for(let r of entry.r_ele)
+                {
+                    let reading = r.reb;
+                    let freq = undefined;
+                    if(settings.freqlist_mode == 1)
+                        freq = freq_vns;
+                    else
+                        freq = freq_narou;
+                    if(freq[reading])
+                    {
+                        for(let possibility of freq[reading])
+                        {
+                            if(seen.has(possibility[0]))
+                            {
+                                if(reading == possibility[0] && entry.k_ele)
+                                    continue;
+                                if(entry.freq.length == 0)
+                                    entry.freq = [possibility[0], reading, possibility[1], possibility[2]];
+                                else
+                                {
+                                    if(entry.freq[2] > possibility[1]) // worse rank
+                                        entry.freq = [possibility[0], reading, possibility[1], possibility[2]];
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+    return ret;
 }
 
 // Skip JMdict entries reappearing in alternative lookups (so only the first one is shown)

etc/process freqlist.py

@@ -0,0 +1,43 @@
+#!python
+import sys
+import json
+
+
+def replace_kata_with_hira(text):
+    newtext = ""
+    for c in text:
+        codepoint = ord(c)
+        if codepoint >= 0x30A0 and codepoint <= 0x30FF:
+            codepoint -= (0x30A0 - 0x3040)
+        newtext += chr(codepoint)
+    return newtext
+
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+reading_mapping = {}
+
+def add_reading_mapping(reading, etc):
+    if reading not in reading_mapping:
+        reading_mapping[reading] = [etc]
+    else:
+        reading_mapping[reading] += [etc]
+
+with open(sys.argv[1], "r", encoding='utf-8') as f:
+    rank = 0
+    for line in f:
+        rank += 1
+        fields = line.split("\t")
+        freq = float(fields[0])
+        lemma_spelling = fields[5].split("-")[0]
+        lemma_reading = replace_kata_with_hira(fields[6])
+        extra_fields = len(fields)-9
+        extra_stuff = extra_fields//7
+        add_reading_mapping(lemma_reading, [lemma_spelling, rank, freq])
+        for i in range(extra_stuff):
+            spelling = fields[9+i*7+1].split("-")[0]
+            reading = fields[9+i*7+2]
+            if reading == lemma_reading:
+                add_reading_mapping(lemma_reading, [spelling, rank, freq])
+        
+print(json.dumps(reading_mapping, ensure_ascii=False, indent=0))

options.js

@@ -169,6 +169,13 @@ function defaults()
         default: false,
         label: "Only look up text when mousing over selection (only when something is selected, setting ignored otherwise)"
     });
+    settings.push({
+        id: "freqlist_mode",
+        kind: "combobox",
+        options: ["None", "Visual Novels", "Narou"],
+        default: 1,
+        label: "Frequency data source"
+    });
     
     // display
     settings.push({

texthook.js

@@ -727,6 +727,7 @@ function build_div_inner(text, result, moreText, index, first_of_many = false)
             }
         }
         
+        
         // for mining
         
         let reading_data_holder = document.createElement("div");
@@ -984,6 +985,19 @@ function build_div_inner(text, result, moreText, index, first_of_many = false)
                 temptag.appendChild(makespan(")"));
         }
         container.appendChild(temptag);
+        
+        if(term.freq && term.freq.length > 0)
+        {
+            let vn_freq_data = document.createElement("span");
+            vn_freq_data.style = "font-size: 80%; margin-top: 4px; opacity: 0.7; margin-left: 5px;"
+            if(term.freq[0] != term.freq[1])
+                vn_freq_data.innerText = "#" + term.freq[2] + " (" + term.freq[0] + ":" + term.freq[1] + ")";
+            else
+                vn_freq_data.innerText = "#" + term.freq[2] + " (" + term.freq[0] + ")";
+            vn_freq_data.innerText += " (" + Math.round(term.freq[3]*100)/100 + "ppm)";
+            container.appendChild(vn_freq_data);
+        }
+        
         container.appendChild(document.createElement("br"));
         
         let definition = document.createElement("div");