Files
VianoBugAnalysis/src/viano_bug_analysis/tools/context_expansion.py
T

160 lines
9.2 KiB
Python

"""Bounded traversal of operator-declared data relationships, never diagnosis."""
import copy
import hashlib
import json
import re
from viano_bug_analysis.lifecycle.storage import json_bytes
from .config import ToolError
class ContextExpansion:
def __init__(self, service, settings):
from .sql_database import is_table_reference
self.service = service
self.rules = settings.get("relationships", [])
self.text_rules = settings.get("texts", [])
self.name_rules = [r for r in settings.get("entities", []) if "name_column" in r]
for rule in self.name_rules:
if (any(not isinstance(rule.get(k),str) or not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,63}",rule[k])
for k in ('entity_type','column','name_column'))
or not is_table_reference(rule.get('table'))
or not isinstance(rule.get('columns'),list)
or not {rule['column'],rule['name_column']} <= set(rule['columns'])):
raise ToolError("PROJECT_ENRICHMENT_CONFIG")
self.max_queries = settings.get("max_queries", 40)
self.max_depth = settings.get("max_depth", 2)
self.max_relationships = settings.get("max_relationships", 128)
if any(type(v) is not int or v <= 0 for v in (self.max_queries, self.max_depth, self.max_relationships)):
raise ToolError("PROJECT_ENRICHMENT_CONFIG")
def same_table(self,first,second):
if first==second:return True
provider=self.service.database
if not callable(getattr(provider,'resolve_table',None)):return False
try:return provider.resolve_table(first)==provider.resolve_table(second)
except ToolError:return False
def collect(self, queries):
queue = [(copy.deepcopy(q), 0) for q in queries]
seen, edges, entities = set(), [], []
additions, facts, context = {}, [], {"database": [], "source": []}
omitted, depth_limited, omitted_relationships = 0, 0, 0
while queue:
query, depth = queue.pop(0)
query_key = json_bytes(query)
if query_key in seen:
continue
seen.add(query_key)
if len(additions) >= self.max_queries:
omitted += 1
continue
data = self.service.collect(query)
capture = json.loads(data)
ref = "research/project/" + hashlib.sha256(data).hexdigest() + "/capture.json"
additions[ref] = data
role = "database" if query["kind"].startswith("database.") else "source"
context[role].append(ref)
visible_result = copy.deepcopy(capture["result"])
descriptions = []
names = []
if query["kind"] == "database.rows":
for rule in self.name_rules:
if not self.same_table(rule['table'],query['table']):
continue
for index, item in enumerate(capture["result"]["items"]):
row = item.get("row", {})
identity, name = row.get(rule["column"]), row.get(rule["name_column"])
if (type(identity) not in (str, int) or not str(identity)
or not isinstance(name, str) or not name.strip()):
continue
names.append({"name":"project_entity_name", "value":{
"type":rule["entity_type"], "id":str(identity), "name":name,
"provenance":{"source_ref":ref,
"json_pointer":f"/result/items/{index}/row/{rule['name_column']}",
"id_pointer":f"/result/items/{index}/row/{rule['column']}"},
"meaning":"explicit_typed_name_binding_not_role_or_causation"}})
if query["kind"] == "database.rows":
for rule in self.text_rules:
if not self.same_table(rule['table'],query['table']):
continue
for i, item in enumerate(capture["result"]["items"]):
row = item.get("row", {})
texts = []
for column in rule["columns"]:
text = row.get(column)
if not isinstance(text, str) or not text.strip():
continue
origin = {"source_ref":ref, "json_pointer":f"/result/items/{i}/row/{column}"}
texts.append({"column":column, "text":text, "origin":origin})
# The raw text is archived once and presented once to the model.
visible_result["items"][i]["row"][column] = {"text_ref":origin}
if texts:
descriptions.append({"name":"project_description", "value":{
"context_role":rule["context_role"], "table":query["table"],
"entity_id":row.get(rule.get("id_column","")), "texts":texts,
"meaning":"source_description_not_verified_runtime_behavior"}})
facts.append({"name": "project_query", "value": {
"query": query, "origin": capture["origin"], "result": visible_result, "capture_ref": ref}})
facts.extend(descriptions)
facts.extend(names)
if query["kind"] != "database.rows":
continue
for rule in self.rules:
if not self.same_table(rule['from_table'],query['table']):
continue
for i, item in enumerate(capture["result"]["items"]):
row = item.get("row", {})
if "when_column" in rule and row.get(rule["when_column"]) != rule["when_value"]:
continue
value = row.get(rule["from_column"])
if value in rule.get("ignore_values", [None, "", 0]) or type(value) is bool:
continue
if not isinstance(value, (str, int)):
raise ToolError("PROJECT_ENRICHMENT_REFERENCE")
if len(edges) >= self.max_relationships:
omitted_relationships += 1
continue
origin = {"source_ref": ref, "json_pointer": f"/result/items/{i}/row/{rule['from_column']}"}
target_type = rule.get("entity_type")
if target_type:
# A reference identifies a related entity, not a runtime observation
# or a proven cause. Never transfer the captured NPC's role to it.
entity = {"type": target_type, "id": str(value),
"roles": [rule["relation"]], "origin": origin}
if entity not in entities:
entities.append(entity)
source_id = row.get(rule.get("from_id_column", ""))
edge = {"relation": rule["relation"], "from_table": query["table"],
"from_id": source_id, "from_column": rule["from_column"],
"target_type": target_type, "target_value": value, "origin": origin,
"meaning": "explicit_data_reference_not_causation"}
if edge not in edges:
edges.append(edge)
if depth >= self.max_depth:
depth_limited += 1
continue
if rule.get("kind", "database.rows") == "source.symbol":
if not isinstance(value, str) or not re.fullmatch("[A-Za-z_][A-Za-z0-9_]*", value):
continue
following = {"kind": "source.symbol", "text": value,
"path_prefix": rule.get("path_prefix", ""), "case_sensitive": True,
"offset": 0, "limit": rule.get("limit", 5)}
else:
following = {"kind": "database.rows", "table": rule["table"],
"columns": rule["columns"], "filters": [{"column": rule["column"], "operator": "eq", "value": value}],
"offset": 0, "limit": rule.get("limit", 5)}
following["filters"].extend(copy.deepcopy(rule.get("filters", [])))
queue.append((following, depth + 1))
for edge in edges:
facts.append({"name": "project_relationship", "value": edge, "origin": edge["origin"]})
if self.rules or omitted or depth_limited or omitted_relationships:
facts.append({"name": "project_context_coverage", "value": {
"queried": len(additions), "max_queries": self.max_queries, "max_depth": self.max_depth,
"omitted_queries": omitted, "depth_limited_relationships": depth_limited,
"omitted_relationships": omitted_relationships, "max_relationships": self.max_relationships,
"meaning": "bounded_context_collection_not_exhaustive_investigation"}})
return additions, facts, context, entities