openforce/custom_objects/formula_engine.py
Goose 4def894801
Some checks failed
Build, Test, and Deploy / build-and-deploy (push) Failing after 29s
feat(formulas): implement safe AST formula execution engine and pre-save evaluation signals (Sprint 4.5A & 4.5B)
- Add zero-trust AST-based formula engine in custom_objects/formula_engine.py supporting arithmetic, logical, string, and date operations.
- Enforce strict AST whitelisting and attribute/function safety checks.
- Add cycle detection and topological sorting of formula dependencies.
- Extend CustomFieldDefinition model with formula_expression and formula_return_type fields.
- Wire pre-save signals to evaluate formula fields across standard models and CustomRecord.
- Expose AST syntax validator endpoint at POST /api/custom-fields/validate-formula/.
- Add comprehensive test suite in tests/test_formula_engine.py with 100% pass rate.
2026-09-27 07:17:04 +00:00

379 lines
13 KiB
Python

import ast
import re
from datetime import date, datetime
from decimal import Decimal
from django.core.exceptions import ValidationError
from django.utils import timezone
ALLOWED_AST_NODES = (
ast.Expression,
ast.BinOp,
ast.UnaryOp,
ast.Add,
ast.Sub,
ast.Mult,
ast.Div,
ast.Mod,
ast.USub,
ast.UAdd,
ast.Compare,
ast.Eq,
ast.NotEq,
ast.Lt,
ast.LtE,
ast.Gt,
ast.GtE,
ast.BoolOp,
ast.And,
ast.Or,
ast.Not,
ast.Constant,
ast.Name,
ast.Attribute,
ast.Call,
ast.Load,
)
def fn_if(condition, true_val, false_val):
return true_val if condition else false_val
def fn_case(*args):
if len(args) < 3:
raise ValueError('CASE requires at least an expression, value, result, and default.')
target = args[0]
pairs = args[1:-1]
default = args[-1]
for i in range(0, len(pairs), 2):
if i + 1 < len(pairs) and target == pairs[i]:
return pairs[i + 1]
return default
def fn_isblank(val):
return val is None or val == '' or val == []
def fn_round(val, digits=0):
if val is None:
return 0
return round(float(val), int(digits))
def fn_today():
return timezone.now().date()
def fn_now():
return timezone.now()
BUILTIN_FUNCTIONS = {
'IF': fn_if,
'CASE': fn_case,
'ISBLANK': fn_isblank,
'NOT': lambda x: not bool(x),
'AND': lambda *args: all(args),
'OR': lambda *args: any(args),
'CONCAT': lambda *args: ''.join(str(a) for a in args if a is not None),
'UPPER': lambda x: str(x).upper() if x is not None else '',
'LOWER': lambda x: str(x).lower() if x is not None else '',
'LEN': lambda x: len(str(x)) if x is not None else 0,
'TRIM': lambda x: str(x).strip() if x is not None else '',
'ROUND': fn_round,
'ABS': lambda x: abs(x) if x is not None else 0,
'MAX': lambda *args: max(args) if args else None,
'MIN': lambda *args: min(args) if args else None,
'TODAY': fn_today,
'NOW': fn_now,
}
class ASTSecurityValidator(ast.NodeVisitor):
def __init__(self):
self.identifiers = set()
def generic_visit(self, node):
if not isinstance(node, ALLOWED_AST_NODES):
raise ValidationError(f'Disallowed expression syntax: {type(node).__name__}')
super().generic_visit(node)
def visit_Call(self, node):
func_name = None
if isinstance(node.func, ast.Name):
func_name = node.func.id
elif isinstance(node.func, ast.Attribute):
func_name = node.func.attr
if not func_name or func_name.upper() not in BUILTIN_FUNCTIONS:
raise ValidationError(f"Function '{func_name}' is not permitted in formula expressions.")
super().generic_visit(node)
def visit_Name(self, node):
if node.id.startswith('__'):
raise ValidationError("Private/dunder attributes are forbidden.")
if not re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*$', node.id):
raise ValidationError(f'Invalid identifier: {node.id}')
if node.id.upper() not in BUILTIN_FUNCTIONS:
self.identifiers.add(node.id)
super().generic_visit(node)
def visit_Attribute(self, node):
if node.attr.startswith('__'):
raise ValidationError("Private/dunder attributes are forbidden.")
parts = []
curr = node
while isinstance(curr, ast.Attribute):
if curr.attr.startswith('__'):
raise ValidationError("Private/dunder attributes are forbidden.")
parts.append(curr.attr)
curr = curr.value
if isinstance(curr, ast.Name):
if curr.id.startswith('__'):
raise ValidationError("Private/dunder attributes are forbidden.")
parts.append(curr.id)
parts.reverse()
full_path = '.'.join(parts)
self.identifiers.add(full_path)
else:
raise ValidationError('Dynamic attribute chaining is not permitted.')
super().generic_visit(node)
def extract_formula_identifiers(expression):
if not expression or not expression.strip():
return set()
parsed = ast.parse(expression.strip(), mode='eval')
validator = ASTSecurityValidator()
validator.visit(parsed)
return validator.identifiers
def validate_formula_syntax(expression):
if not expression or not expression.strip():
raise ValidationError('Formula expression cannot be empty.')
try:
parsed = ast.parse(expression.strip(), mode='eval')
except SyntaxError as e:
raise ValidationError(f'Invalid formula syntax: {e.msg}')
validator = ASTSecurityValidator()
validator.visit(parsed)
return True
def topological_sort_formulas(formula_dict):
"""
Sort formula field names topologically based on their dependencies on other formula fields in the dict.
formula_dict: dict of {field_name: formula_expression}
Raises ValidationError on cycle.
"""
deps = {}
in_degree = {name: 0 for name in formula_dict}
graph = {name: [] for name in formula_dict}
for name, expr in formula_dict.items():
try:
ids = extract_formula_identifiers(expr)
except Exception:
ids = set()
# Dependencies that are also formula fields
f_deps = {dep for dep in ids if dep in formula_dict and dep != name}
deps[name] = f_deps
for dep in f_deps:
graph[dep].append(name)
in_degree[name] += 1
queue = [name for name, deg in in_degree.items() if deg == 0]
sorted_fields = []
while queue:
curr = queue.pop(0)
sorted_fields.append(curr)
for neighbor in graph[curr]:
in_degree[neighbor] -= 1
if in_degree[neighbor] == 0:
queue.append(neighbor)
if len(sorted_fields) != len(formula_dict):
raise ValidationError("Circular dependency detected in formula fields.")
return sorted_fields
def resolve_field_value(record, path):
if not path:
return None
parts = path.split('.')
curr = record
for part in parts:
if curr is None:
return None
if isinstance(curr, dict):
curr = curr.get(part)
elif hasattr(curr, 'custom_fields') and isinstance(curr.custom_fields, dict) and part in curr.custom_fields:
curr = curr.custom_fields[part]
elif hasattr(curr, 'data') and isinstance(curr.data, dict) and part in curr.data:
curr = curr.data[part]
elif hasattr(curr, part):
val = getattr(curr, part)
curr = val() if callable(val) else val
else:
return None
return curr
class ASTFormulaEvaluator(ast.NodeVisitor):
def __init__(self, record, context=None):
self.record = record
self.context = context or {}
def evaluate(self, expression):
parsed = ast.parse(expression.strip(), mode='eval')
validator = ASTSecurityValidator()
validator.visit(parsed)
return self.visit(parsed.body)
def visit_Constant(self, node):
return node.value
def visit_Name(self, node):
name = node.id
if name.upper() in BUILTIN_FUNCTIONS:
return BUILTIN_FUNCTIONS[name.upper()]
if name in self.context:
return self.context[name]
return resolve_field_value(self.record, name)
def visit_Attribute(self, node):
parts = []
curr = node
while isinstance(curr, ast.Attribute):
parts.append(curr.attr)
curr = curr.value
if isinstance(curr, ast.Name):
parts.append(curr.id)
parts.reverse()
full_path = '.'.join(parts)
if full_path in self.context:
return self.context[full_path]
return resolve_field_value(self.record, full_path)
raise ValidationError('Invalid attribute evaluation.')
def visit_BinOp(self, node):
left = self.visit(node.left)
right = self.visit(node.right)
if isinstance(node.op, ast.Add):
if isinstance(left, str) or isinstance(right, str):
return str(left or '') + str(right or '')
return (left or 0) + (right or 0)
elif isinstance(node.op, ast.Sub):
return (left or 0) - (right or 0)
elif isinstance(node.op, ast.Mult):
return (left or 0) * (right or 0)
elif isinstance(node.op, ast.Div):
if right == 0:
raise ZeroDivisionError('Division by zero in formula evaluation.')
return (left or 0) / right
elif isinstance(node.op, ast.Mod):
return (left or 0) % right
raise ValidationError(f'Unsupported binary operator: {type(node.op).__name__}')
def visit_UnaryOp(self, node):
operand = self.visit(node.operand)
if isinstance(node.op, ast.USub):
return -operand
elif isinstance(node.op, ast.UAdd):
return +operand
elif isinstance(node.op, ast.Not):
return not operand
raise ValidationError(f'Unsupported unary operator: {type(node.op).__name__}')
def visit_Compare(self, node):
left = self.visit(node.left)
for op, comparator in zip(node.ops, node.comparators):
right = self.visit(comparator)
if isinstance(op, ast.Eq) and not (left == right):
return False
elif isinstance(op, ast.NotEq) and not (left != right):
return False
elif isinstance(op, ast.Lt) and not (left < right):
return False
elif isinstance(op, ast.LtE) and not (left <= right):
return False
elif isinstance(op, ast.Gt) and not (left > right):
return False
elif isinstance(op, ast.GtE) and not (left >= right):
return False
left = right
return True
def visit_BoolOp(self, node):
if isinstance(node.op, ast.And):
for val_node in node.values:
if not self.visit(val_node):
return False
return True
elif isinstance(node.op, ast.Or):
for val_node in node.values:
if self.visit(val_node):
return True
return False
raise ValidationError(f'Unsupported boolean operator: {type(node.op).__name__}')
def visit_Call(self, node):
func = self.visit(node.func)
if not callable(func):
raise ValidationError(f'Expression {node.func} is not a valid function.')
args = [self.visit(arg) for arg in node.args]
return func(*args)
def evaluate_formula(record, expression, return_type='text', context=None):
if not expression or not expression.strip():
return None
evaluator = ASTFormulaEvaluator(record, context=context)
val = evaluator.evaluate(expression)
if val is None:
return None
if return_type in ('number', 'percent'):
return float(val)
elif return_type == 'currency':
return round(float(val), 2)
elif return_type in ('boolean', 'checkbox'):
return bool(val)
elif return_type in ('text', 'string'):
return str(val)
return val
def evaluate_record_formulas(record, entity_name=None):
from custom_objects.models import CustomFieldDefinition
is_custom_record = hasattr(record, 'custom_object_id') and record.custom_object_id
if entity_name:
fields = CustomFieldDefinition.objects.filter(entity_name=entity_name, field_type='formula')
elif is_custom_record:
fields = CustomFieldDefinition.objects.filter(custom_object_id=record.custom_object_id, field_type='formula')
elif hasattr(record, '_meta') and record._meta.model_name:
# Standard model fallback by class name
fields = CustomFieldDefinition.objects.filter(entity_name=record.__class__.__name__, field_type='formula')
else:
return
if not fields.exists():
return
if is_custom_record:
if record.data is None:
record.data = {}
storage = record.data
else:
if not hasattr(record, 'custom_fields') or record.custom_fields is None:
record.custom_fields = {}
storage = record.custom_fields
formula_dict = {f.name: f.formula_expression for f in fields if f.formula_expression}
if not formula_dict:
return
try:
execution_order = topological_sort_formulas(formula_dict)
except ValidationError:
execution_order = list(formula_dict.keys())
field_map = {f.name: f for f in fields}
for field_name in execution_order:
f = field_map[field_name]
try:
res = evaluate_formula(record, f.formula_expression, return_type=getattr(f, 'formula_return_type', 'text'))
storage[field_name] = res
except Exception:
pass
def evaluate_all_formulas_for_instance(instance):
evaluate_record_formulas(instance)