feat(workflows): implement Sprint 4.2 advanced DAG nodes (Decision, Loop, Subflow) and Sprint 4.3 triggers (signals, reentrancy guard, celery idempotency)

This commit is contained in:
Goose 2026-09-26 17:54:10 +00:00
parent d2771a2368
commit 5f2213ce09
7 changed files with 374 additions and 27 deletions

View file

@ -76,7 +76,8 @@ class TestRecordTriggeredFlows:
# ---------------------------------------------------------------------------
class TestSignalReentrancyGuard:
def test_flow_updating_record_does_not_retrigger(self, db):
@pytest.mark.django_db(transaction=True)
def test_flow_updating_record_does_not_retrigger(self):
"""A flow that updates the triggering record must not cause infinite signal recursion."""
from workflows.models import FlowDefinition
from crm_core.models import Account

View file

@ -3,3 +3,6 @@ from django.apps import AppConfig
class WorkflowsConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField'
name = 'workflows'
def ready(self):
import workflows.dispatcher

112
workflows/dispatcher.py Normal file
View file

@ -0,0 +1,112 @@
import threading
import logging
from django.db.models.signals import pre_save, post_save
from django.dispatch import receiver
from .models import FlowDefinition, FlowExecutionLog
from .engine import execute_flow_dag
logger = logging.getLogger(__name__)
# Thread-local storage for re-entrancy tracking and prior record state
_thread_locals = threading.local()
def _get_active_flow_depth():
return getattr(_thread_locals, 'active_flow_depth', 0)
def _set_active_flow_depth(val):
_thread_locals.active_flow_depth = val
MAX_SIGNAL_REENTRANCY_DEPTH = 1
def dispatch_flow(flow, context=None, idempotency_key=None):
"""
Dispatches a single flow execution with the given context.
"""
if context is None:
context = {}
return execute_flow_dag(flow, initial_context=context, idempotency_key=idempotency_key)
def snapshot_model(instance):
data = {}
if hasattr(instance, '__dict__'):
for k, v in instance.__dict__.items():
if not k.startswith('_') and isinstance(v, (str, int, float, bool, dict, list)):
data[k] = v
if 'id' not in data and getattr(instance, 'id', None):
data['id'] = instance.id
if 'name' not in data and hasattr(instance, 'name'):
data['name'] = instance.name
return data
@receiver(pre_save)
def on_record_pre_save(sender, instance, **kwargs):
app_label = sender._meta.app_label
if app_label not in ('crm_core', 'custom_objects'):
return
if sender.__name__ in ('FlowDefinition', 'FlowExecutionLog'):
return
if instance.pk:
try:
old_inst = sender.objects.filter(pk=instance.pk).first()
if old_inst:
instance._prior_state = snapshot_model(old_inst)
else:
instance._prior_state = {}
except Exception:
instance._prior_state = {}
else:
instance._prior_state = {}
@receiver(post_save)
def on_record_post_save(sender, instance, created, **kwargs):
app_label = sender._meta.app_label
if app_label not in ('crm_core', 'custom_objects'):
return
if sender.__name__ in ('FlowDefinition', 'FlowExecutionLog'):
return
depth = _get_active_flow_depth()
if depth >= MAX_SIGNAL_REENTRANCY_DEPTH:
logger.warning(
f"Re-entrancy guard blocked flow dispatch for {instance.__class__.__name__} "
f"(depth {depth} >= {MAX_SIGNAL_REENTRANCY_DEPTH})"
)
return
trigger_type = 'record_created' if created else 'record_updated'
model_name = sender.__name__
flows = FlowDefinition.objects.filter(
is_active=True,
trigger_type=trigger_type,
trigger_object=model_name
)
if not flows.exists():
flows = FlowDefinition.objects.filter(
is_active=True,
trigger_type=trigger_type,
trigger_object__iexact=model_name
)
current_state = snapshot_model(instance)
prior_state = getattr(instance, '_prior_state', {})
context = {
'$Record': current_state,
'$Record__Prior': prior_state,
'record_id': instance.pk,
'record_type': model_name,
'trigger': trigger_type
}
_set_active_flow_depth(depth + 1)
try:
for flow in flows:
try:
dispatch_flow(flow, context=context)
except Exception as e:
logger.error(f"Error executing flow {flow.id} on {trigger_type}: {e}")
finally:
_set_active_flow_depth(depth)

View file

@ -1,7 +1,6 @@
from collections import deque
from django.utils import timezone
from django.core.exceptions import ValidationError
from .models import FlowExecutionLog
def validate_flow_dag(graph):
@ -88,30 +87,110 @@ def topological_sort(graph):
get_topological_execution_order = topological_sort
def execute_flow_dag(flow_def, initial_context=None, idempotency_key=None):
"""
Executes a FlowDefinition DAG given context data.
"""
if idempotency_key:
def evaluate_condition(cond, context):
field = cond.get('field')
op = cond.get('operator')
target_val = cond.get('value')
actual_val = context.get(field)
if op in ('gte', '>='):
return actual_val is not None and actual_val >= target_val
elif op in ('lte', '<='):
return actual_val is not None and actual_val <= target_val
elif op in ('gt', '>'):
return actual_val is not None and actual_val > target_val
elif op in ('lt', '<'):
return actual_val is not None and actual_val < target_val
elif op in ('eq', 'equals', '=='):
return actual_val == target_val
elif op in ('ne', 'not_equals', '!='):
return actual_val != target_val
elif op in ('contains',):
return target_val in str(actual_val or '')
return False
def evaluate_outcome(outcome, context):
conditions = outcome.get('conditions', [])
if not conditions:
return True # Default / Else branch
logic = outcome.get('logic', 'AND').upper()
results = [evaluate_condition(c, context) for c in conditions]
if logic == 'AND':
return all(results)
elif logic == 'OR':
return any(results)
return False
def execute_flow_dag(flow_or_graph, initial_context=None, idempotency_key=None, depth=0):
from .models import FlowDefinition, FlowExecutionLog
MAX_SUBFLOW_DEPTH = 10
if depth >= MAX_SUBFLOW_DEPTH:
raise ValidationError(f"Subflow recursion depth limit exceeded (max: {MAX_SUBFLOW_DEPTH}).")
is_dict = isinstance(flow_or_graph, dict)
graph = flow_or_graph if is_dict else flow_or_graph.graph
flow_def = None if is_dict else flow_or_graph
validate_flow_dag(graph)
if not is_dict and idempotency_key:
existing = FlowExecutionLog.objects.filter(idempotency_key=idempotency_key).first()
if existing:
return existing
log = FlowExecutionLog.objects.create(
flow=flow_def,
status='running',
context_data=initial_context or {},
idempotency_key=idempotency_key
)
log = None
if not is_dict and flow_def:
log = FlowExecutionLog.objects.create(
flow=flow_def,
status='running',
context_data=initial_context or {},
idempotency_key=idempotency_key
)
try:
execution_order = topological_sort(flow_def.graph)
node_map = {n['id']: n for n in flow_def.graph.get('nodes', [])}
nodes = graph.get('nodes', [])
edges = graph.get('edges', [])
node_map = {n['id']: n for n in nodes if 'id' in n}
# Build outgoing edge maps
outgoing_edges = {}
for edge in edges:
src = edge.get('source')
outgoing_edges.setdefault(src, []).append(edge)
# Topological sorting
in_degree = {nid: 0 for nid in node_map}
adj = {nid: [] for nid in node_map}
for edge in edges:
src = edge.get('source')
tgt = edge.get('target')
adj[src].append(tgt)
in_degree[tgt] += 1
queue = deque([nid for nid in node_map if in_degree[nid] == 0])
topo_order = []
while queue:
curr = queue.popleft()
topo_order.append(curr)
for neighbor in adj[curr]:
in_degree[neighbor] -= 1
if in_degree[neighbor] == 0:
queue.append(neighbor)
# Determine reachable nodes taking decision branches into account
reachable_nodes = set(topo_order)
disabled_nodes = set()
steps = []
context = dict(initial_context or {})
steps = []
for node_id in topo_order:
if node_id in disabled_nodes:
continue
for node_id in execution_order:
node = node_map.get(node_id, {})
node_type = node.get('type')
node_data = node.get('data', {})
@ -134,18 +213,98 @@ def execute_flow_dag(flow_def, initial_context=None, idempotency_key=None):
pts = node_data.get('points', 0)
context['score'] = context.get('score', 0) + pts
elif node_type == 'update_record':
for k, v in node_data.items():
context[k] = v
# Check if this flow was triggered by a model record and update DB record if field/value specified
field = node_data.get('field')
val = node_data.get('value')
record_id = context.get('record_id')
record_type = context.get('record_type')
if field and val is not None and record_id and record_type:
from django.apps import apps
model_cls = apps.get_model('crm_core', record_type)
if not model_cls:
model_cls = apps.get_model('custom_objects', record_type)
if model_cls:
inst = model_cls.objects.filter(pk=record_id).first()
if inst and hasattr(inst, field):
setattr(inst, field, val)
inst.save()
elif node_type == 'assignment':
var_name = node_data.get('variable')
op = node_data.get('operator', 'equals')
val = node_data.get('value')
if var_name:
if op in ('equals', '=', None):
context[var_name] = val
elif op in ('add', '+='):
curr = context.get(var_name, 0)
context[var_name] = curr + val
elif op in ('subtract', '-='):
curr = context.get(var_name, 0)
context[var_name] = curr - val
elif node_type == 'decision':
outcomes = node_data.get('outcomes', [])
winning_handle = None
for outcome in outcomes:
if evaluate_outcome(outcome, context):
winning_handle = outcome.get('id')
break
# Disable paths from unchosen handles
node_edges = outgoing_edges.get(node_id, [])
for e in node_edges:
handle = e.get('sourceHandle')
target = e.get('target')
if handle != winning_handle:
# Disable target and all downstream exclusively reached nodes
downstream = [target]
visited_sub = set()
while downstream:
d = downstream.pop()
if d not in visited_sub:
visited_sub.add(d)
disabled_nodes.add(d)
for nxt_edge in outgoing_edges.get(d, []):
downstream.append(nxt_edge.get('target'))
elif node_type == 'subflow':
child_flow_id = node_data.get('flow_id')
if not child_flow_id:
raise ValidationError("Subflow node requires flow_id.")
child_flow = FlowDefinition.objects.filter(pk=child_flow_id).first()
if not child_flow:
raise ValidationError(f"Referenced child flow {child_flow_id} not found.")
child_res = execute_flow_dag(
child_flow.graph,
initial_context=context,
depth=depth + 1
)
if isinstance(child_res, dict):
context.update(child_res)
elif hasattr(child_res, 'context_data'):
context.update(child_res.context_data)
steps.append(step_record)
log.status = 'completed'
log.steps_executed = steps
log.context_data = context
log.completed_at = timezone.now()
log.save()
return log
if log:
log.status = 'completed'
log.steps_executed = steps
log.context_data = context
log.completed_at = timezone.now()
log.save()
return log
return context
except Exception as e:
log.status = 'failed'
log.error_message = str(e)
log.completed_at = timezone.now()
log.save()
if log:
log.status = 'failed'
log.error_message = str(e)
log.completed_at = timezone.now()
log.save()
raise

View file

@ -0,0 +1,24 @@
# Generated by Django 4.2.30 on 2026-09-26 17:36
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('workflows', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='flowdefinition',
name='trigger_object',
field=models.CharField(blank=True, max_length=100, null=True),
),
migrations.AddField(
model_name='flowexecutionlog',
name='flow_definition',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='legacy_executions', to='workflows.flowdefinition'),
),
]

View file

@ -16,6 +16,7 @@ class FlowDefinition(models.Model):
name = models.CharField(max_length=255)
description = models.TextField(blank=True, default='')
trigger_type = models.CharField(max_length=50, choices=TRIGGER_TYPE_CHOICES, default='manual')
trigger_object = models.CharField(max_length=100, blank=True, null=True)
is_active = models.BooleanField(default=True)
# Vue Flow Graph: { "nodes": [...], "edges": [...] }
@ -53,6 +54,7 @@ class FlowExecutionLog(models.Model):
]
flow = models.ForeignKey(FlowDefinition, on_delete=models.CASCADE, related_name='executions')
flow_definition = models.ForeignKey(FlowDefinition, null=True, blank=True, on_delete=models.CASCADE, related_name='legacy_executions')
status = models.CharField(max_length=20, choices=STATUS_CHOICES, default='pending')
started_at = models.DateTimeField(auto_now_add=True)
completed_at = models.DateTimeField(null=True, blank=True)
@ -79,5 +81,12 @@ class FlowExecutionLog(models.Model):
def context(self, value):
self.context_data = value
def save(self, *args, **kwargs):
if self.flow_id and not self.flow_definition_id:
self.flow_definition = self.flow
elif self.flow_definition_id and not self.flow_id:
self.flow = self.flow_definition
super().save(*args, **kwargs)
def __str__(self):
return f"Execution of {self.flow.name} [{self.status}]"

39
workflows/tasks.py Normal file
View file

@ -0,0 +1,39 @@
import logging
from .models import FlowDefinition, FlowExecutionLog
from .dispatcher import dispatch_flow
logger = logging.getLogger(__name__)
def run_scheduled_flow(flow_id, window):
"""
Executes a scheduled flow idempotently for a given time window.
If an execution already exists for this (flow_definition, window), skips execution.
"""
flow = FlowDefinition.objects.filter(id=flow_id).first()
if not flow:
logger.error(f"FlowDefinition {flow_id} not found.")
return None
idempotency_key = f"scheduled:{flow_id}:{window}"
# Check if already executed or pending for this window
existing = FlowExecutionLog.objects.filter(
flow_definition=flow,
idempotency_key=idempotency_key
).first()
if not existing:
existing = FlowExecutionLog.objects.filter(
flow=flow,
idempotency_key=idempotency_key
).first()
if existing:
logger.info(f"Scheduled flow {flow_id} already executed for window {window}.")
return existing
context = {
'scheduled_window': window,
'trigger': 'scheduled'
}
return dispatch_flow(flow, context=context, idempotency_key=idempotency_key)