From 447098783002d36a6f36236be18143507bd8a52f Mon Sep 17 00:00:00 2001
From: wareya <wareya@gmail.com>
Date: Wed, 13 Dec 2017 02:13:09 -0500
Subject: [PATCH] split jmdict into multiple <4MB files

---
 background-script.js | 116 ++++++++++++++++++++++++++++---------------
 manifest.json        |  11 +++-
 readme.txt           |   2 +-
 3 files changed, 87 insertions(+), 42 deletions(-)

diff --git a/background-script.js b/background-script.js
index ba16050..8690d17 100644
--- a/background-script.js
+++ b/background-script.js
@@ -8,60 +8,96 @@
 // We have to send an HTTP request and store the result in a string before parsing it into an object.
 // Seriously.
 
-let dict = undefined;
+let dict = [];
 let lookup_kan = new Map();
 let lookup_kana = new Map();
 
+let dictsloaded = 0;
+
 function builddict()
 {
     if (this.readyState === 4)
     {
         if (this.status === 200)
         {
-            dict = JSON.parse(this.responseText);
-            
-            // Build map of spellings to dictionary entry.
-
-            for (let i = 0; i < dict.length; i++)
-            {
-                let entry = dict[i];
-                if (entry.k_ele != undefined) for (let j = 0; j < entry.k_ele.length; j++)
-                {
-                    let s = entry.k_ele[j];
-                    if (!lookup_kan.has(s.keb))
-                        lookup_kan.set(s.keb, [i]);
-                    else
-                        lookup_kan.get(s.keb).push(i);
-                }
-                for (let j = 0; j < entry.r_ele.length; j++)
-                {
-                    let s = entry.r_ele[j];
-                    if (!lookup_kana.has(s.reb))
-                        lookup_kana.set(s.reb, [i]);
-                    else
-                        lookup_kana.get(s.reb).push(i);
-                }
-            }
-            for (let i = 0; i < 5; i++)
-            {
-                console.log(dict[i]);
-            }
-            console.log("built lookup tables");
-            console.log("size:");
-            console.log(Object.keys(lookup_kan).length);
-            console.log(Object.keys(lookup_kana).length);
-            console.log("dict size:");
-            console.log(dict.length);
+            dict = dict.concat(JSON.parse(this.responseText));
+            dictsloaded++;
         }
         else
             console.error(xhr.statusText);
     }
 }
-let req = new XMLHttpRequest();
-req.addEventListener("load", builddict);
-// FIXME: apparently this has to be in an async function now.
-req.open("GET", browser.extension.getURL("dict/JMdict.json"));
-req.send();
+
+// Having a >4MB flat text file with stupid adhoc formatting is okay
+// but having a >4MB json file with nothing weird in it isn't
+// Thanks, linter team!
+function load_part_of_dictionary(filename)
+{
+    let req = new XMLHttpRequest();
+    req.addEventListener("load", builddict);
+    // FIXME: apparently this has to be in an async function now.
+    req.open("GET", browser.extension.getURL(filename));
+    req.send();
+    return req;
+}
+load_part_of_dictionary("dict/JMdict1.json");
+load_part_of_dictionary("dict/JMdict2.json");
+load_part_of_dictionary("dict/JMdict3.json");
+load_part_of_dictionary("dict/JMdict4.json");
+load_part_of_dictionary("dict/JMdict5.json");
+load_part_of_dictionary("dict/JMdict6.json");
+load_part_of_dictionary("dict/JMdict7.json");
+load_part_of_dictionary("dict/JMdict8.json");
+load_part_of_dictionary("dict/JMdict9.json");
+load_part_of_dictionary("dict/JMdict10.json");
+
+function buildlookups()
+{
+    // Build map of spellings to dictionary entry.
+
+    for (let i = 0; i < dict.length; i++)
+    {
+        let entry = dict[i];
+        if (entry.k_ele != undefined) for (let j = 0; j < entry.k_ele.length; j++)
+        {
+            let s = entry.k_ele[j];
+            if (!lookup_kan.has(s.keb))
+                lookup_kan.set(s.keb, [i]);
+            else
+                lookup_kan.get(s.keb).push(i);
+        }
+        for (let j = 0; j < entry.r_ele.length; j++)
+        {
+            let s = entry.r_ele[j];
+            if (!lookup_kana.has(s.reb))
+                lookup_kana.set(s.reb, [i]);
+            else
+                lookup_kana.get(s.reb).push(i);
+        }
+    }
+    for (let i = 0; i < 5; i++)
+    {
+        console.log(dict[i]);
+    }
+    console.log("built lookup tables");
+    console.log("size:");
+    console.log(Object.keys(lookup_kan).length);
+    console.log(Object.keys(lookup_kana).length);
+    console.log("dict size:");
+    console.log(dict.length);
+}
+
+let waiter = undefined;
+function waittobuildlookups()
+{
+    if(dictsloaded == 10)
+    {
+        clearInterval(waiter);
+        waiter = undefined;
+        buildlookups();
+    }
+}
+waiter = setInterval(waittobuildlookups, 1000);
 
 // In JMdict, part-of-speech tags are XML entities.
 // We processed JMdict's XML with entity processing disabled so we can just use the bare tags (e.g. "v1", not "ichidan verb").
diff --git a/manifest.json b/manifest.json
index b44ab8f..bb80576 100644
--- a/manifest.json
+++ b/manifest.json
@@ -18,7 +18,16 @@
     },
 
     "web_accessible_resources": [
-        "dict/JMdict.json",
+        "dict/JMdict1.json",
+        "dict/JMdict2.json",
+        "dict/JMdict3.json",
+        "dict/JMdict4.json",
+        "dict/JMdict5.json",
+        "dict/JMdict6.json",
+        "dict/JMdict7.json",
+        "dict/JMdict8.json",
+        "dict/JMdict9.json",
+        "dict/JMdict10.json",
         "reader.js",
         "reader.html",
         "etc/popupbase.css"
diff --git a/readme.txt b/readme.txt
index 779ceb3..c664a9f 100644
--- a/readme.txt
+++ b/readme.txt
@@ -6,7 +6,7 @@ This is my first nontrivial javascript work and there are probably a lot of obvi
 
 Building:
 
-Build process requires python and lxml, or an existing copy of the extension to rip JMdict.json from.
+Build process requires python and lxml, or an existing copy of the extension to rip JMdict#.json from.
 
 - Download JMdict.gz from http://www.edrdg.org/jmdict/edict_doc.html
 - Convert it to json with https://gist.github.com/wareya/c2175520db5f1927e4f6ba839487dd8c