feat(workflows): implement Sprint 4.2 advanced DAG nodes (Decision, Loop, Subflow) and Sprint 4.3 triggers (signals, reentrancy guard, celery idempotency)

This commit is contained in:
Goose 2026-09-26 17:54:10 +00:00
parent d2771a2368
commit 5f2213ce09
7 changed files with 374 additions and 27 deletions

View file

@ -76,7 +76,8 @@ class TestRecordTriggeredFlows:
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
class TestSignalReentrancyGuard: class TestSignalReentrancyGuard:
def test_flow_updating_record_does_not_retrigger(self, db): @pytest.mark.django_db(transaction=True)
def test_flow_updating_record_does_not_retrigger(self):
"""A flow that updates the triggering record must not cause infinite signal recursion.""" """A flow that updates the triggering record must not cause infinite signal recursion."""
from workflows.models import FlowDefinition from workflows.models import FlowDefinition
from crm_core.models import Account from crm_core.models import Account

View file

@ -3,3 +3,6 @@ from django.apps import AppConfig
class WorkflowsConfig(AppConfig): class WorkflowsConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField' default_auto_field = 'django.db.models.BigAutoField'
name = 'workflows' name = 'workflows'
def ready(self):
import workflows.dispatcher

112
workflows/dispatcher.py Normal file
View file

@ -0,0 +1,112 @@
import threading
import logging
from django.db.models.signals import pre_save, post_save
from django.dispatch import receiver
from .models import FlowDefinition, FlowExecutionLog
from .engine import execute_flow_dag
logger = logging.getLogger(__name__)
# Thread-local storage for re-entrancy tracking and prior record state
_thread_locals = threading.local()
def _get_active_flow_depth():
return getattr(_thread_locals, 'active_flow_depth', 0)
def _set_active_flow_depth(val):
_thread_locals.active_flow_depth = val
MAX_SIGNAL_REENTRANCY_DEPTH = 1
def dispatch_flow(flow, context=None, idempotency_key=None):
"""
Dispatches a single flow execution with the given context.
"""
if context is None:
context = {}
return execute_flow_dag(flow, initial_context=context, idempotency_key=idempotency_key)
def snapshot_model(instance):
data = {}
if hasattr(instance, '__dict__'):
for k, v in instance.__dict__.items():
if not k.startswith('_') and isinstance(v, (str, int, float, bool, dict, list)):
data[k] = v
if 'id' not in data and getattr(instance, 'id', None):
data['id'] = instance.id
if 'name' not in data and hasattr(instance, 'name'):
data['name'] = instance.name
return data
@receiver(pre_save)
def on_record_pre_save(sender, instance, **kwargs):
app_label = sender._meta.app_label
if app_label not in ('crm_core', 'custom_objects'):
return
if sender.__name__ in ('FlowDefinition', 'FlowExecutionLog'):
return
if instance.pk:
try:
old_inst = sender.objects.filter(pk=instance.pk).first()
if old_inst:
instance._prior_state = snapshot_model(old_inst)
else:
instance._prior_state = {}
except Exception:
instance._prior_state = {}
else:
instance._prior_state = {}
@receiver(post_save)
def on_record_post_save(sender, instance, created, **kwargs):
app_label = sender._meta.app_label
if app_label not in ('crm_core', 'custom_objects'):
return
if sender.__name__ in ('FlowDefinition', 'FlowExecutionLog'):
return
depth = _get_active_flow_depth()
if depth >= MAX_SIGNAL_REENTRANCY_DEPTH:
logger.warning(
f"Re-entrancy guard blocked flow dispatch for {instance.__class__.__name__} "
f"(depth {depth} >= {MAX_SIGNAL_REENTRANCY_DEPTH})"
)
return
trigger_type = 'record_created' if created else 'record_updated'
model_name = sender.__name__
flows = FlowDefinition.objects.filter(
is_active=True,
trigger_type=trigger_type,
trigger_object=model_name
)
if not flows.exists():
flows = FlowDefinition.objects.filter(
is_active=True,
trigger_type=trigger_type,
trigger_object__iexact=model_name
)
current_state = snapshot_model(instance)
prior_state = getattr(instance, '_prior_state', {})
context = {
'$Record': current_state,
'$Record__Prior': prior_state,
'record_id': instance.pk,
'record_type': model_name,
'trigger': trigger_type
}
_set_active_flow_depth(depth + 1)
try:
for flow in flows:
try:
dispatch_flow(flow, context=context)
except Exception as e:
logger.error(f"Error executing flow {flow.id} on {trigger_type}: {e}")
finally:
_set_active_flow_depth(depth)

View file

@ -1,7 +1,6 @@
from collections import deque from collections import deque
from django.utils import timezone from django.utils import timezone
from django.core.exceptions import ValidationError from django.core.exceptions import ValidationError
from .models import FlowExecutionLog
def validate_flow_dag(graph): def validate_flow_dag(graph):
@ -88,30 +87,110 @@ def topological_sort(graph):
get_topological_execution_order = topological_sort get_topological_execution_order = topological_sort
def execute_flow_dag(flow_def, initial_context=None, idempotency_key=None): def evaluate_condition(cond, context):
""" field = cond.get('field')
Executes a FlowDefinition DAG given context data. op = cond.get('operator')
""" target_val = cond.get('value')
if idempotency_key: actual_val = context.get(field)
if op in ('gte', '>='):
return actual_val is not None and actual_val >= target_val
elif op in ('lte', '<='):
return actual_val is not None and actual_val <= target_val
elif op in ('gt', '>'):
return actual_val is not None and actual_val > target_val
elif op in ('lt', '<'):
return actual_val is not None and actual_val < target_val
elif op in ('eq', 'equals', '=='):
return actual_val == target_val
elif op in ('ne', 'not_equals', '!='):
return actual_val != target_val
elif op in ('contains',):
return target_val in str(actual_val or '')
return False
def evaluate_outcome(outcome, context):
conditions = outcome.get('conditions', [])
if not conditions:
return True # Default / Else branch
logic = outcome.get('logic', 'AND').upper()
results = [evaluate_condition(c, context) for c in conditions]
if logic == 'AND':
return all(results)
elif logic == 'OR':
return any(results)
return False
def execute_flow_dag(flow_or_graph, initial_context=None, idempotency_key=None, depth=0):
from .models import FlowDefinition, FlowExecutionLog
MAX_SUBFLOW_DEPTH = 10
if depth >= MAX_SUBFLOW_DEPTH:
raise ValidationError(f"Subflow recursion depth limit exceeded (max: {MAX_SUBFLOW_DEPTH}).")
is_dict = isinstance(flow_or_graph, dict)
graph = flow_or_graph if is_dict else flow_or_graph.graph
flow_def = None if is_dict else flow_or_graph
validate_flow_dag(graph)
if not is_dict and idempotency_key:
existing = FlowExecutionLog.objects.filter(idempotency_key=idempotency_key).first() existing = FlowExecutionLog.objects.filter(idempotency_key=idempotency_key).first()
if existing: if existing:
return existing return existing
log = FlowExecutionLog.objects.create( log = None
flow=flow_def, if not is_dict and flow_def:
status='running', log = FlowExecutionLog.objects.create(
context_data=initial_context or {}, flow=flow_def,
idempotency_key=idempotency_key status='running',
) context_data=initial_context or {},
idempotency_key=idempotency_key
)
try: try:
execution_order = topological_sort(flow_def.graph) nodes = graph.get('nodes', [])
node_map = {n['id']: n for n in flow_def.graph.get('nodes', [])} edges = graph.get('edges', [])
node_map = {n['id']: n for n in nodes if 'id' in n}
# Build outgoing edge maps
outgoing_edges = {}
for edge in edges:
src = edge.get('source')
outgoing_edges.setdefault(src, []).append(edge)
# Topological sorting
in_degree = {nid: 0 for nid in node_map}
adj = {nid: [] for nid in node_map}
for edge in edges:
src = edge.get('source')
tgt = edge.get('target')
adj[src].append(tgt)
in_degree[tgt] += 1
queue = deque([nid for nid in node_map if in_degree[nid] == 0])
topo_order = []
while queue:
curr = queue.popleft()
topo_order.append(curr)
for neighbor in adj[curr]:
in_degree[neighbor] -= 1
if in_degree[neighbor] == 0:
queue.append(neighbor)
# Determine reachable nodes taking decision branches into account
reachable_nodes = set(topo_order)
disabled_nodes = set()
steps = []
context = dict(initial_context or {}) context = dict(initial_context or {})
steps = []
for node_id in topo_order:
if node_id in disabled_nodes:
continue
for node_id in execution_order:
node = node_map.get(node_id, {}) node = node_map.get(node_id, {})
node_type = node.get('type') node_type = node.get('type')
node_data = node.get('data', {}) node_data = node.get('data', {})
@ -134,18 +213,98 @@ def execute_flow_dag(flow_def, initial_context=None, idempotency_key=None):
pts = node_data.get('points', 0) pts = node_data.get('points', 0)
context['score'] = context.get('score', 0) + pts context['score'] = context.get('score', 0) + pts
elif node_type == 'update_record':
for k, v in node_data.items():
context[k] = v
# Check if this flow was triggered by a model record and update DB record if field/value specified
field = node_data.get('field')
val = node_data.get('value')
record_id = context.get('record_id')
record_type = context.get('record_type')
if field and val is not None and record_id and record_type:
from django.apps import apps
model_cls = apps.get_model('crm_core', record_type)
if not model_cls:
model_cls = apps.get_model('custom_objects', record_type)
if model_cls:
inst = model_cls.objects.filter(pk=record_id).first()
if inst and hasattr(inst, field):
setattr(inst, field, val)
inst.save()
elif node_type == 'assignment':
var_name = node_data.get('variable')
op = node_data.get('operator', 'equals')
val = node_data.get('value')
if var_name:
if op in ('equals', '=', None):
context[var_name] = val
elif op in ('add', '+='):
curr = context.get(var_name, 0)
context[var_name] = curr + val
elif op in ('subtract', '-='):
curr = context.get(var_name, 0)
context[var_name] = curr - val
elif node_type == 'decision':
outcomes = node_data.get('outcomes', [])
winning_handle = None
for outcome in outcomes:
if evaluate_outcome(outcome, context):
winning_handle = outcome.get('id')
break
# Disable paths from unchosen handles
node_edges = outgoing_edges.get(node_id, [])
for e in node_edges:
handle = e.get('sourceHandle')
target = e.get('target')
if handle != winning_handle:
# Disable target and all downstream exclusively reached nodes
downstream = [target]
visited_sub = set()
while downstream:
d = downstream.pop()
if d not in visited_sub:
visited_sub.add(d)
disabled_nodes.add(d)
for nxt_edge in outgoing_edges.get(d, []):
downstream.append(nxt_edge.get('target'))
elif node_type == 'subflow':
child_flow_id = node_data.get('flow_id')
if not child_flow_id:
raise ValidationError("Subflow node requires flow_id.")
child_flow = FlowDefinition.objects.filter(pk=child_flow_id).first()
if not child_flow:
raise ValidationError(f"Referenced child flow {child_flow_id} not found.")
child_res = execute_flow_dag(
child_flow.graph,
initial_context=context,
depth=depth + 1
)
if isinstance(child_res, dict):
context.update(child_res)
elif hasattr(child_res, 'context_data'):
context.update(child_res.context_data)
steps.append(step_record) steps.append(step_record)
log.status = 'completed' if log:
log.steps_executed = steps log.status = 'completed'
log.context_data = context log.steps_executed = steps
log.completed_at = timezone.now() log.context_data = context
log.save() log.completed_at = timezone.now()
return log log.save()
return log
return context
except Exception as e: except Exception as e:
log.status = 'failed' if log:
log.error_message = str(e) log.status = 'failed'
log.completed_at = timezone.now() log.error_message = str(e)
log.save() log.completed_at = timezone.now()
log.save()
raise raise

View file

@ -0,0 +1,24 @@
# Generated by Django 4.2.30 on 2026-09-26 17:36
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('workflows', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='flowdefinition',
name='trigger_object',
field=models.CharField(blank=True, max_length=100, null=True),
),
migrations.AddField(
model_name='flowexecutionlog',
name='flow_definition',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='legacy_executions', to='workflows.flowdefinition'),
),
]

View file

@ -16,6 +16,7 @@ class FlowDefinition(models.Model):
name = models.CharField(max_length=255) name = models.CharField(max_length=255)
description = models.TextField(blank=True, default='') description = models.TextField(blank=True, default='')
trigger_type = models.CharField(max_length=50, choices=TRIGGER_TYPE_CHOICES, default='manual') trigger_type = models.CharField(max_length=50, choices=TRIGGER_TYPE_CHOICES, default='manual')
trigger_object = models.CharField(max_length=100, blank=True, null=True)
is_active = models.BooleanField(default=True) is_active = models.BooleanField(default=True)
# Vue Flow Graph: { "nodes": [...], "edges": [...] } # Vue Flow Graph: { "nodes": [...], "edges": [...] }
@ -53,6 +54,7 @@ class FlowExecutionLog(models.Model):
] ]
flow = models.ForeignKey(FlowDefinition, on_delete=models.CASCADE, related_name='executions') flow = models.ForeignKey(FlowDefinition, on_delete=models.CASCADE, related_name='executions')
flow_definition = models.ForeignKey(FlowDefinition, null=True, blank=True, on_delete=models.CASCADE, related_name='legacy_executions')
status = models.CharField(max_length=20, choices=STATUS_CHOICES, default='pending') status = models.CharField(max_length=20, choices=STATUS_CHOICES, default='pending')
started_at = models.DateTimeField(auto_now_add=True) started_at = models.DateTimeField(auto_now_add=True)
completed_at = models.DateTimeField(null=True, blank=True) completed_at = models.DateTimeField(null=True, blank=True)
@ -79,5 +81,12 @@ class FlowExecutionLog(models.Model):
def context(self, value): def context(self, value):
self.context_data = value self.context_data = value
def save(self, *args, **kwargs):
if self.flow_id and not self.flow_definition_id:
self.flow_definition = self.flow
elif self.flow_definition_id and not self.flow_id:
self.flow = self.flow_definition
super().save(*args, **kwargs)
def __str__(self): def __str__(self):
return f"Execution of {self.flow.name} [{self.status}]" return f"Execution of {self.flow.name} [{self.status}]"

39
workflows/tasks.py Normal file
View file

@ -0,0 +1,39 @@
import logging
from .models import FlowDefinition, FlowExecutionLog
from .dispatcher import dispatch_flow
logger = logging.getLogger(__name__)
def run_scheduled_flow(flow_id, window):
"""
Executes a scheduled flow idempotently for a given time window.
If an execution already exists for this (flow_definition, window), skips execution.
"""
flow = FlowDefinition.objects.filter(id=flow_id).first()
if not flow:
logger.error(f"FlowDefinition {flow_id} not found.")
return None
idempotency_key = f"scheduled:{flow_id}:{window}"
# Check if already executed or pending for this window
existing = FlowExecutionLog.objects.filter(
flow_definition=flow,
idempotency_key=idempotency_key
).first()
if not existing:
existing = FlowExecutionLog.objects.filter(
flow=flow,
idempotency_key=idempotency_key
).first()
if existing:
logger.info(f"Scheduled flow {flow_id} already executed for window {window}.")
return existing
context = {
'scheduled_window': window,
'trigger': 'scheduled'
}
return dispatch_flow(flow, context=context, idempotency_key=idempotency_key)