Add shared read-only Git and database research tools

scripts/dev_world_db.py

@@ -0,0 +1,114 @@
+"""Development-only provisioner. Separate from the read-only analysis runtime.
+
+Uses a portable MariaDB, binds loopback, restores only the named World section.
+Never connects to any other database server. Runtime credentials have SELECT only.
+"""
+import argparse
+import hashlib
+import io
+import json
+from pathlib import Path
+import secrets
+import re
+import subprocess
+import time
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument("command", choices=["start", "restore", "stop"])
+    p.add_argument("--binary-dir", type=Path, required=True)
+    p.add_argument("--directory", type=Path, required=True)
+    p.add_argument("--dump", type=Path)
+    p.add_argument("--database", default="cata_world")
+    p.add_argument("--port", type=int, default=33079)
+    args = p.parse_args()
+    import pymysql
+    import zstandard
+    root = args.directory.resolve()
+    root.mkdir(parents=True, exist_ok=True)
+    binary = args.binary_dir.resolve()
+    def check_local(connection):
+        with connection.cursor() as cursor:
+            cursor.execute("SELECT @@datadir")
+            actual = Path(cursor.fetchone()[0]).resolve()
+        if actual != (root/"data").resolve():
+            raise RuntimeError("Port belongs to another data directory; refusing administration")
+    state_file = root / "credentials.local.json"
+    if not state_file.exists():
+        state_file.write_text(json.dumps({"root":secrets.token_urlsafe(32), "reader":secrets.token_urlsafe(32)}), encoding="utf-8")
+    credentials = json.loads(state_file.read_text(encoding="utf-8"))
+    if args.command == "start":
+        config = root / "server.ini"
+        config.write_text("[mysqld]\n" + f"basedir={binary.parent.as_posix()}\ndatadir={(root/'data').as_posix()}\n"
+                          + f"port={args.port}\nbind-address=127.0.0.1\nmax_allowed_packet=256M\n"
+                          + "local_infile=0\nread_only=ON\nskip-log-bin\n" + f"log-error={(root/'server.log').as_posix()}\n", encoding="utf-8")
+        process = subprocess.Popen([str(binary/"mariadbd.exe"), "--defaults-file="+str(config)],
+            creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0), stdin=subprocess.DEVNULL,
+            stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        (root / "server.pid.json").write_text(json.dumps({"pid":process.pid, "port":args.port}), encoding="utf-8")
+        for _ in range(60):
+            try:
+                connection = pymysql.connect(host="127.0.0.1",port=args.port,user="root", password="",connect_timeout=1)
+                check_local(connection)
+                with connection.cursor() as cursor:
+                    cursor.execute("SET PASSWORD FOR 'root'@'localhost' = PASSWORD(%s)", (credentials["root"],))
+                connection.close()
+                print(json.dumps({"status":"STARTED", "port":args.port}))
+                return
+            except pymysql.err.OperationalError as error:
+                if error.args[0] == 1045:
+                    break
+                if process.poll() is not None:
+                    raise RuntimeError("Local MariaDB failed to start; see server.log") from None
+                time.sleep(.5)
+    connection = pymysql.connect(host="127.0.0.1",port=args.port,user="root",password=credentials["root"],connect_timeout=3,autocommit=True)
+    try:
+        check_local(connection)
+        if args.command == "stop":
+            with connection.cursor() as cursor:
+                cursor.execute("SHUTDOWN")
+            print(json.dumps({"status":"STOPPED"}))
+            return
+        if args.command == "start":
+            print(json.dumps({"status":"STARTED", "port":args.port}))
+            return
+        if args.dump is None or not re.fullmatch("[A-Za-z_][A-Za-z0-9_]{0,63}",args.database):
+            raise ValueError("A dump and plain database identifier are required")
+        client_config = root / "restore.local.ini"
+        client_config.write_text(f"[client]\nhost=127.0.0.1\nport={args.port}\nuser=root\npassword={credentials['root']}\n",encoding="utf-8")
+        error_log = (root / "restore.log").open("wb")
+        process = subprocess.Popen([str(binary/"mariadb.exe"),"--defaults-file="+str(client_config),"--binary-mode","--max-allowed-packet=256M"],
+                                   stdin=subprocess.PIPE,stdout=subprocess.DEVNULL,stderr=error_log,
+                                   creationflags=getattr(subprocess,"CREATE_NO_WINDOW",0))
+        selected = False
+        with args.dump.open("rb") as raw:
+            stream = io.TextIOWrapper(zstandard.ZstdDecompressor().stream_reader(raw),encoding="utf-8")
+            for line in stream:
+                if line.startswith("-- Current Database:"):
+                    selected = line.strip() == f"-- Current Database: `{args.database}`"
+                # Header sets the dump encoding and SQL modes; subsequent sections
+                # are discarded completely, including account/character content.
+                if selected or (not selected and line.startswith("/*!40101 SET NAMES")):
+                    process.stdin.write(line.encode("utf-8"))
+        process.stdin.close()
+        code = process.wait()
+        error_log.close()
+        if code:
+            raise RuntimeError("World restore failed; see restore.log")
+        with connection.cursor() as cursor:
+            cursor.execute("CREATE USER IF NOT EXISTS 'viano_reader'@'127.0.0.1' IDENTIFIED BY %s", (credentials["reader"],))
+            cursor.execute("GRANT SELECT ON `"+args.database+"`.* TO 'viano_reader'@'127.0.0.1'")
+            cursor.execute("SET GLOBAL read_only=ON")
+            cursor.execute("SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=%s", (args.database,))
+            count = cursor.fetchone()[0]
+        with args.dump.open("rb") as dump_stream:
+            sha = hashlib.file_digest(dump_stream,"sha256").hexdigest()
+        (root/"snapshot.json").write_text(json.dumps({"database":args.database,"backup_sha256":sha,"tables":count}),encoding="utf-8")
+        print(json.dumps({"status":"RESTORED", "database":args.database,"tables":count,"backup_sha256":sha}))
+    finally:
+        connection.close()
+
+
+if __name__ == "__main__":
+    main()