160 lines
9.2 KiB
Python
160 lines
9.2 KiB
Python
"""Bounded traversal of operator-declared data relationships, never diagnosis."""
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
import re
|
|
|
|
from viano_bug_analysis.lifecycle.storage import json_bytes
|
|
from .config import ToolError
|
|
|
|
|
|
class ContextExpansion:
|
|
def __init__(self, service, settings):
|
|
from .sql_database import is_table_reference
|
|
self.service = service
|
|
self.rules = settings.get("relationships", [])
|
|
self.text_rules = settings.get("texts", [])
|
|
self.name_rules = [r for r in settings.get("entities", []) if "name_column" in r]
|
|
for rule in self.name_rules:
|
|
if (any(not isinstance(rule.get(k),str) or not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,63}",rule[k])
|
|
for k in ('entity_type','column','name_column'))
|
|
or not is_table_reference(rule.get('table'))
|
|
or not isinstance(rule.get('columns'),list)
|
|
or not {rule['column'],rule['name_column']} <= set(rule['columns'])):
|
|
raise ToolError("PROJECT_ENRICHMENT_CONFIG")
|
|
|
|
self.max_queries = settings.get("max_queries", 40)
|
|
self.max_depth = settings.get("max_depth", 2)
|
|
self.max_relationships = settings.get("max_relationships", 128)
|
|
if any(type(v) is not int or v <= 0 for v in (self.max_queries, self.max_depth, self.max_relationships)):
|
|
raise ToolError("PROJECT_ENRICHMENT_CONFIG")
|
|
|
|
def same_table(self,first,second):
|
|
if first==second:return True
|
|
provider=self.service.database
|
|
if not callable(getattr(provider,'resolve_table',None)):return False
|
|
try:return provider.resolve_table(first)==provider.resolve_table(second)
|
|
except ToolError:return False
|
|
|
|
def collect(self, queries):
|
|
queue = [(copy.deepcopy(q), 0) for q in queries]
|
|
seen, edges, entities = set(), [], []
|
|
additions, facts, context = {}, [], {"database": [], "source": []}
|
|
omitted, depth_limited, omitted_relationships = 0, 0, 0
|
|
while queue:
|
|
query, depth = queue.pop(0)
|
|
query_key = json_bytes(query)
|
|
if query_key in seen:
|
|
continue
|
|
seen.add(query_key)
|
|
if len(additions) >= self.max_queries:
|
|
omitted += 1
|
|
continue
|
|
data = self.service.collect(query)
|
|
capture = json.loads(data)
|
|
ref = "research/project/" + hashlib.sha256(data).hexdigest() + "/capture.json"
|
|
additions[ref] = data
|
|
role = "database" if query["kind"].startswith("database.") else "source"
|
|
context[role].append(ref)
|
|
visible_result = copy.deepcopy(capture["result"])
|
|
descriptions = []
|
|
names = []
|
|
if query["kind"] == "database.rows":
|
|
for rule in self.name_rules:
|
|
if not self.same_table(rule['table'],query['table']):
|
|
continue
|
|
for index, item in enumerate(capture["result"]["items"]):
|
|
row = item.get("row", {})
|
|
identity, name = row.get(rule["column"]), row.get(rule["name_column"])
|
|
if (type(identity) not in (str, int) or not str(identity)
|
|
or not isinstance(name, str) or not name.strip()):
|
|
continue
|
|
names.append({"name":"project_entity_name", "value":{
|
|
"type":rule["entity_type"], "id":str(identity), "name":name,
|
|
"provenance":{"source_ref":ref,
|
|
"json_pointer":f"/result/items/{index}/row/{rule['name_column']}",
|
|
"id_pointer":f"/result/items/{index}/row/{rule['column']}"},
|
|
"meaning":"explicit_typed_name_binding_not_role_or_causation"}})
|
|
|
|
if query["kind"] == "database.rows":
|
|
for rule in self.text_rules:
|
|
if not self.same_table(rule['table'],query['table']):
|
|
continue
|
|
for i, item in enumerate(capture["result"]["items"]):
|
|
row = item.get("row", {})
|
|
texts = []
|
|
for column in rule["columns"]:
|
|
text = row.get(column)
|
|
if not isinstance(text, str) or not text.strip():
|
|
continue
|
|
origin = {"source_ref":ref, "json_pointer":f"/result/items/{i}/row/{column}"}
|
|
texts.append({"column":column, "text":text, "origin":origin})
|
|
# The raw text is archived once and presented once to the model.
|
|
visible_result["items"][i]["row"][column] = {"text_ref":origin}
|
|
if texts:
|
|
descriptions.append({"name":"project_description", "value":{
|
|
"context_role":rule["context_role"], "table":query["table"],
|
|
"entity_id":row.get(rule.get("id_column","")), "texts":texts,
|
|
"meaning":"source_description_not_verified_runtime_behavior"}})
|
|
facts.append({"name": "project_query", "value": {
|
|
"query": query, "origin": capture["origin"], "result": visible_result, "capture_ref": ref}})
|
|
facts.extend(descriptions)
|
|
facts.extend(names)
|
|
if query["kind"] != "database.rows":
|
|
continue
|
|
for rule in self.rules:
|
|
if not self.same_table(rule['from_table'],query['table']):
|
|
continue
|
|
for i, item in enumerate(capture["result"]["items"]):
|
|
row = item.get("row", {})
|
|
if "when_column" in rule and row.get(rule["when_column"]) != rule["when_value"]:
|
|
continue
|
|
value = row.get(rule["from_column"])
|
|
if value in rule.get("ignore_values", [None, "", 0]) or type(value) is bool:
|
|
continue
|
|
if not isinstance(value, (str, int)):
|
|
raise ToolError("PROJECT_ENRICHMENT_REFERENCE")
|
|
if len(edges) >= self.max_relationships:
|
|
omitted_relationships += 1
|
|
continue
|
|
origin = {"source_ref": ref, "json_pointer": f"/result/items/{i}/row/{rule['from_column']}"}
|
|
target_type = rule.get("entity_type")
|
|
if target_type:
|
|
# A reference identifies a related entity, not a runtime observation
|
|
# or a proven cause. Never transfer the captured NPC's role to it.
|
|
entity = {"type": target_type, "id": str(value),
|
|
"roles": [rule["relation"]], "origin": origin}
|
|
if entity not in entities:
|
|
entities.append(entity)
|
|
source_id = row.get(rule.get("from_id_column", ""))
|
|
edge = {"relation": rule["relation"], "from_table": query["table"],
|
|
"from_id": source_id, "from_column": rule["from_column"],
|
|
"target_type": target_type, "target_value": value, "origin": origin,
|
|
"meaning": "explicit_data_reference_not_causation"}
|
|
if edge not in edges:
|
|
edges.append(edge)
|
|
if depth >= self.max_depth:
|
|
depth_limited += 1
|
|
continue
|
|
if rule.get("kind", "database.rows") == "source.symbol":
|
|
if not isinstance(value, str) or not re.fullmatch("[A-Za-z_][A-Za-z0-9_]*", value):
|
|
continue
|
|
following = {"kind": "source.symbol", "text": value,
|
|
"path_prefix": rule.get("path_prefix", ""), "case_sensitive": True,
|
|
"offset": 0, "limit": rule.get("limit", 5)}
|
|
else:
|
|
following = {"kind": "database.rows", "table": rule["table"],
|
|
"columns": rule["columns"], "filters": [{"column": rule["column"], "operator": "eq", "value": value}],
|
|
"offset": 0, "limit": rule.get("limit", 5)}
|
|
following["filters"].extend(copy.deepcopy(rule.get("filters", [])))
|
|
queue.append((following, depth + 1))
|
|
for edge in edges:
|
|
facts.append({"name": "project_relationship", "value": edge, "origin": edge["origin"]})
|
|
if self.rules or omitted or depth_limited or omitted_relationships:
|
|
facts.append({"name": "project_context_coverage", "value": {
|
|
"queried": len(additions), "max_queries": self.max_queries, "max_depth": self.max_depth,
|
|
"omitted_queries": omitted, "depth_limited_relationships": depth_limited,
|
|
"omitted_relationships": omitted_relationships, "max_relationships": self.max_relationships,
|
|
"meaning": "bounded_context_collection_not_exhaustive_investigation"}})
|
|
return additions, facts, context, entities
|