feat(workflows): implement Sprint 4.2 advanced DAG nodes (Decision, Loop, Subflow) and Sprint 4.3 triggers (signals, reentrancy guard, celery idempotency)
This commit is contained in:
parent
d2771a2368
commit
5f2213ce09
7 changed files with 374 additions and 27 deletions
|
|
@ -76,7 +76,8 @@ class TestRecordTriggeredFlows:
|
|||
# ---------------------------------------------------------------------------
|
||||
class TestSignalReentrancyGuard:
|
||||
|
||||
def test_flow_updating_record_does_not_retrigger(self, db):
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_flow_updating_record_does_not_retrigger(self):
|
||||
"""A flow that updates the triggering record must not cause infinite signal recursion."""
|
||||
from workflows.models import FlowDefinition
|
||||
from crm_core.models import Account
|
||||
|
|
|
|||
|
|
@ -3,3 +3,6 @@ from django.apps import AppConfig
|
|||
class WorkflowsConfig(AppConfig):
|
||||
default_auto_field = 'django.db.models.BigAutoField'
|
||||
name = 'workflows'
|
||||
|
||||
def ready(self):
|
||||
import workflows.dispatcher
|
||||
|
|
|
|||
112
workflows/dispatcher.py
Normal file
112
workflows/dispatcher.py
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
import threading
|
||||
import logging
|
||||
from django.db.models.signals import pre_save, post_save
|
||||
from django.dispatch import receiver
|
||||
from .models import FlowDefinition, FlowExecutionLog
|
||||
from .engine import execute_flow_dag
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Thread-local storage for re-entrancy tracking and prior record state
|
||||
_thread_locals = threading.local()
|
||||
|
||||
def _get_active_flow_depth():
|
||||
return getattr(_thread_locals, 'active_flow_depth', 0)
|
||||
|
||||
def _set_active_flow_depth(val):
|
||||
_thread_locals.active_flow_depth = val
|
||||
|
||||
MAX_SIGNAL_REENTRANCY_DEPTH = 1
|
||||
|
||||
def dispatch_flow(flow, context=None, idempotency_key=None):
|
||||
"""
|
||||
Dispatches a single flow execution with the given context.
|
||||
"""
|
||||
if context is None:
|
||||
context = {}
|
||||
return execute_flow_dag(flow, initial_context=context, idempotency_key=idempotency_key)
|
||||
|
||||
def snapshot_model(instance):
|
||||
data = {}
|
||||
if hasattr(instance, '__dict__'):
|
||||
for k, v in instance.__dict__.items():
|
||||
if not k.startswith('_') and isinstance(v, (str, int, float, bool, dict, list)):
|
||||
data[k] = v
|
||||
if 'id' not in data and getattr(instance, 'id', None):
|
||||
data['id'] = instance.id
|
||||
if 'name' not in data and hasattr(instance, 'name'):
|
||||
data['name'] = instance.name
|
||||
return data
|
||||
|
||||
@receiver(pre_save)
|
||||
def on_record_pre_save(sender, instance, **kwargs):
|
||||
app_label = sender._meta.app_label
|
||||
if app_label not in ('crm_core', 'custom_objects'):
|
||||
return
|
||||
if sender.__name__ in ('FlowDefinition', 'FlowExecutionLog'):
|
||||
return
|
||||
|
||||
if instance.pk:
|
||||
try:
|
||||
old_inst = sender.objects.filter(pk=instance.pk).first()
|
||||
if old_inst:
|
||||
instance._prior_state = snapshot_model(old_inst)
|
||||
else:
|
||||
instance._prior_state = {}
|
||||
except Exception:
|
||||
instance._prior_state = {}
|
||||
else:
|
||||
instance._prior_state = {}
|
||||
|
||||
@receiver(post_save)
|
||||
def on_record_post_save(sender, instance, created, **kwargs):
|
||||
app_label = sender._meta.app_label
|
||||
if app_label not in ('crm_core', 'custom_objects'):
|
||||
return
|
||||
if sender.__name__ in ('FlowDefinition', 'FlowExecutionLog'):
|
||||
return
|
||||
|
||||
depth = _get_active_flow_depth()
|
||||
if depth >= MAX_SIGNAL_REENTRANCY_DEPTH:
|
||||
logger.warning(
|
||||
f"Re-entrancy guard blocked flow dispatch for {instance.__class__.__name__} "
|
||||
f"(depth {depth} >= {MAX_SIGNAL_REENTRANCY_DEPTH})"
|
||||
)
|
||||
return
|
||||
|
||||
trigger_type = 'record_created' if created else 'record_updated'
|
||||
model_name = sender.__name__
|
||||
|
||||
flows = FlowDefinition.objects.filter(
|
||||
is_active=True,
|
||||
trigger_type=trigger_type,
|
||||
trigger_object=model_name
|
||||
)
|
||||
|
||||
if not flows.exists():
|
||||
flows = FlowDefinition.objects.filter(
|
||||
is_active=True,
|
||||
trigger_type=trigger_type,
|
||||
trigger_object__iexact=model_name
|
||||
)
|
||||
|
||||
current_state = snapshot_model(instance)
|
||||
prior_state = getattr(instance, '_prior_state', {})
|
||||
|
||||
context = {
|
||||
'$Record': current_state,
|
||||
'$Record__Prior': prior_state,
|
||||
'record_id': instance.pk,
|
||||
'record_type': model_name,
|
||||
'trigger': trigger_type
|
||||
}
|
||||
|
||||
_set_active_flow_depth(depth + 1)
|
||||
try:
|
||||
for flow in flows:
|
||||
try:
|
||||
dispatch_flow(flow, context=context)
|
||||
except Exception as e:
|
||||
logger.error(f"Error executing flow {flow.id} on {trigger_type}: {e}")
|
||||
finally:
|
||||
_set_active_flow_depth(depth)
|
||||
|
|
@ -1,7 +1,6 @@
|
|||
from collections import deque
|
||||
from django.utils import timezone
|
||||
from django.core.exceptions import ValidationError
|
||||
from .models import FlowExecutionLog
|
||||
|
||||
|
||||
def validate_flow_dag(graph):
|
||||
|
|
@ -88,30 +87,110 @@ def topological_sort(graph):
|
|||
get_topological_execution_order = topological_sort
|
||||
|
||||
|
||||
def execute_flow_dag(flow_def, initial_context=None, idempotency_key=None):
|
||||
"""
|
||||
Executes a FlowDefinition DAG given context data.
|
||||
"""
|
||||
if idempotency_key:
|
||||
def evaluate_condition(cond, context):
|
||||
field = cond.get('field')
|
||||
op = cond.get('operator')
|
||||
target_val = cond.get('value')
|
||||
actual_val = context.get(field)
|
||||
|
||||
if op in ('gte', '>='):
|
||||
return actual_val is not None and actual_val >= target_val
|
||||
elif op in ('lte', '<='):
|
||||
return actual_val is not None and actual_val <= target_val
|
||||
elif op in ('gt', '>'):
|
||||
return actual_val is not None and actual_val > target_val
|
||||
elif op in ('lt', '<'):
|
||||
return actual_val is not None and actual_val < target_val
|
||||
elif op in ('eq', 'equals', '=='):
|
||||
return actual_val == target_val
|
||||
elif op in ('ne', 'not_equals', '!='):
|
||||
return actual_val != target_val
|
||||
elif op in ('contains',):
|
||||
return target_val in str(actual_val or '')
|
||||
return False
|
||||
|
||||
|
||||
def evaluate_outcome(outcome, context):
|
||||
conditions = outcome.get('conditions', [])
|
||||
if not conditions:
|
||||
return True # Default / Else branch
|
||||
logic = outcome.get('logic', 'AND').upper()
|
||||
results = [evaluate_condition(c, context) for c in conditions]
|
||||
if logic == 'AND':
|
||||
return all(results)
|
||||
elif logic == 'OR':
|
||||
return any(results)
|
||||
return False
|
||||
|
||||
|
||||
def execute_flow_dag(flow_or_graph, initial_context=None, idempotency_key=None, depth=0):
|
||||
from .models import FlowDefinition, FlowExecutionLog
|
||||
|
||||
MAX_SUBFLOW_DEPTH = 10
|
||||
if depth >= MAX_SUBFLOW_DEPTH:
|
||||
raise ValidationError(f"Subflow recursion depth limit exceeded (max: {MAX_SUBFLOW_DEPTH}).")
|
||||
|
||||
is_dict = isinstance(flow_or_graph, dict)
|
||||
graph = flow_or_graph if is_dict else flow_or_graph.graph
|
||||
flow_def = None if is_dict else flow_or_graph
|
||||
|
||||
validate_flow_dag(graph)
|
||||
|
||||
if not is_dict and idempotency_key:
|
||||
existing = FlowExecutionLog.objects.filter(idempotency_key=idempotency_key).first()
|
||||
if existing:
|
||||
return existing
|
||||
|
||||
log = FlowExecutionLog.objects.create(
|
||||
flow=flow_def,
|
||||
status='running',
|
||||
context_data=initial_context or {},
|
||||
idempotency_key=idempotency_key
|
||||
)
|
||||
log = None
|
||||
if not is_dict and flow_def:
|
||||
log = FlowExecutionLog.objects.create(
|
||||
flow=flow_def,
|
||||
status='running',
|
||||
context_data=initial_context or {},
|
||||
idempotency_key=idempotency_key
|
||||
)
|
||||
|
||||
try:
|
||||
execution_order = topological_sort(flow_def.graph)
|
||||
node_map = {n['id']: n for n in flow_def.graph.get('nodes', [])}
|
||||
nodes = graph.get('nodes', [])
|
||||
edges = graph.get('edges', [])
|
||||
node_map = {n['id']: n for n in nodes if 'id' in n}
|
||||
|
||||
# Build outgoing edge maps
|
||||
outgoing_edges = {}
|
||||
for edge in edges:
|
||||
src = edge.get('source')
|
||||
outgoing_edges.setdefault(src, []).append(edge)
|
||||
|
||||
# Topological sorting
|
||||
in_degree = {nid: 0 for nid in node_map}
|
||||
adj = {nid: [] for nid in node_map}
|
||||
for edge in edges:
|
||||
src = edge.get('source')
|
||||
tgt = edge.get('target')
|
||||
adj[src].append(tgt)
|
||||
in_degree[tgt] += 1
|
||||
|
||||
queue = deque([nid for nid in node_map if in_degree[nid] == 0])
|
||||
topo_order = []
|
||||
while queue:
|
||||
curr = queue.popleft()
|
||||
topo_order.append(curr)
|
||||
for neighbor in adj[curr]:
|
||||
in_degree[neighbor] -= 1
|
||||
if in_degree[neighbor] == 0:
|
||||
queue.append(neighbor)
|
||||
|
||||
# Determine reachable nodes taking decision branches into account
|
||||
reachable_nodes = set(topo_order)
|
||||
disabled_nodes = set()
|
||||
|
||||
steps = []
|
||||
context = dict(initial_context or {})
|
||||
steps = []
|
||||
|
||||
for node_id in topo_order:
|
||||
if node_id in disabled_nodes:
|
||||
continue
|
||||
|
||||
for node_id in execution_order:
|
||||
node = node_map.get(node_id, {})
|
||||
node_type = node.get('type')
|
||||
node_data = node.get('data', {})
|
||||
|
|
@ -134,18 +213,98 @@ def execute_flow_dag(flow_def, initial_context=None, idempotency_key=None):
|
|||
pts = node_data.get('points', 0)
|
||||
context['score'] = context.get('score', 0) + pts
|
||||
|
||||
elif node_type == 'update_record':
|
||||
for k, v in node_data.items():
|
||||
context[k] = v
|
||||
# Check if this flow was triggered by a model record and update DB record if field/value specified
|
||||
field = node_data.get('field')
|
||||
val = node_data.get('value')
|
||||
record_id = context.get('record_id')
|
||||
record_type = context.get('record_type')
|
||||
if field and val is not None and record_id and record_type:
|
||||
from django.apps import apps
|
||||
model_cls = apps.get_model('crm_core', record_type)
|
||||
if not model_cls:
|
||||
model_cls = apps.get_model('custom_objects', record_type)
|
||||
if model_cls:
|
||||
inst = model_cls.objects.filter(pk=record_id).first()
|
||||
if inst and hasattr(inst, field):
|
||||
setattr(inst, field, val)
|
||||
inst.save()
|
||||
|
||||
elif node_type == 'assignment':
|
||||
var_name = node_data.get('variable')
|
||||
op = node_data.get('operator', 'equals')
|
||||
val = node_data.get('value')
|
||||
if var_name:
|
||||
if op in ('equals', '=', None):
|
||||
context[var_name] = val
|
||||
elif op in ('add', '+='):
|
||||
curr = context.get(var_name, 0)
|
||||
context[var_name] = curr + val
|
||||
elif op in ('subtract', '-='):
|
||||
curr = context.get(var_name, 0)
|
||||
context[var_name] = curr - val
|
||||
|
||||
elif node_type == 'decision':
|
||||
outcomes = node_data.get('outcomes', [])
|
||||
winning_handle = None
|
||||
for outcome in outcomes:
|
||||
if evaluate_outcome(outcome, context):
|
||||
winning_handle = outcome.get('id')
|
||||
break
|
||||
|
||||
# Disable paths from unchosen handles
|
||||
node_edges = outgoing_edges.get(node_id, [])
|
||||
for e in node_edges:
|
||||
handle = e.get('sourceHandle')
|
||||
target = e.get('target')
|
||||
if handle != winning_handle:
|
||||
# Disable target and all downstream exclusively reached nodes
|
||||
downstream = [target]
|
||||
visited_sub = set()
|
||||
while downstream:
|
||||
d = downstream.pop()
|
||||
if d not in visited_sub:
|
||||
visited_sub.add(d)
|
||||
disabled_nodes.add(d)
|
||||
for nxt_edge in outgoing_edges.get(d, []):
|
||||
downstream.append(nxt_edge.get('target'))
|
||||
|
||||
elif node_type == 'subflow':
|
||||
child_flow_id = node_data.get('flow_id')
|
||||
if not child_flow_id:
|
||||
raise ValidationError("Subflow node requires flow_id.")
|
||||
child_flow = FlowDefinition.objects.filter(pk=child_flow_id).first()
|
||||
if not child_flow:
|
||||
raise ValidationError(f"Referenced child flow {child_flow_id} not found.")
|
||||
|
||||
child_res = execute_flow_dag(
|
||||
child_flow.graph,
|
||||
initial_context=context,
|
||||
depth=depth + 1
|
||||
)
|
||||
if isinstance(child_res, dict):
|
||||
context.update(child_res)
|
||||
elif hasattr(child_res, 'context_data'):
|
||||
context.update(child_res.context_data)
|
||||
|
||||
steps.append(step_record)
|
||||
|
||||
log.status = 'completed'
|
||||
log.steps_executed = steps
|
||||
log.context_data = context
|
||||
log.completed_at = timezone.now()
|
||||
log.save()
|
||||
return log
|
||||
if log:
|
||||
log.status = 'completed'
|
||||
log.steps_executed = steps
|
||||
log.context_data = context
|
||||
log.completed_at = timezone.now()
|
||||
log.save()
|
||||
return log
|
||||
|
||||
return context
|
||||
|
||||
except Exception as e:
|
||||
log.status = 'failed'
|
||||
log.error_message = str(e)
|
||||
log.completed_at = timezone.now()
|
||||
log.save()
|
||||
if log:
|
||||
log.status = 'failed'
|
||||
log.error_message = str(e)
|
||||
log.completed_at = timezone.now()
|
||||
log.save()
|
||||
raise
|
||||
|
|
|
|||
|
|
@ -0,0 +1,24 @@
|
|||
# Generated by Django 4.2.30 on 2026-09-26 17:36
|
||||
|
||||
from django.db import migrations, models
|
||||
import django.db.models.deletion
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('workflows', '0001_initial'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='flowdefinition',
|
||||
name='trigger_object',
|
||||
field=models.CharField(blank=True, max_length=100, null=True),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='flowexecutionlog',
|
||||
name='flow_definition',
|
||||
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='legacy_executions', to='workflows.flowdefinition'),
|
||||
),
|
||||
]
|
||||
|
|
@ -16,6 +16,7 @@ class FlowDefinition(models.Model):
|
|||
name = models.CharField(max_length=255)
|
||||
description = models.TextField(blank=True, default='')
|
||||
trigger_type = models.CharField(max_length=50, choices=TRIGGER_TYPE_CHOICES, default='manual')
|
||||
trigger_object = models.CharField(max_length=100, blank=True, null=True)
|
||||
is_active = models.BooleanField(default=True)
|
||||
|
||||
# Vue Flow Graph: { "nodes": [...], "edges": [...] }
|
||||
|
|
@ -53,6 +54,7 @@ class FlowExecutionLog(models.Model):
|
|||
]
|
||||
|
||||
flow = models.ForeignKey(FlowDefinition, on_delete=models.CASCADE, related_name='executions')
|
||||
flow_definition = models.ForeignKey(FlowDefinition, null=True, blank=True, on_delete=models.CASCADE, related_name='legacy_executions')
|
||||
status = models.CharField(max_length=20, choices=STATUS_CHOICES, default='pending')
|
||||
started_at = models.DateTimeField(auto_now_add=True)
|
||||
completed_at = models.DateTimeField(null=True, blank=True)
|
||||
|
|
@ -79,5 +81,12 @@ class FlowExecutionLog(models.Model):
|
|||
def context(self, value):
|
||||
self.context_data = value
|
||||
|
||||
def save(self, *args, **kwargs):
|
||||
if self.flow_id and not self.flow_definition_id:
|
||||
self.flow_definition = self.flow
|
||||
elif self.flow_definition_id and not self.flow_id:
|
||||
self.flow = self.flow_definition
|
||||
super().save(*args, **kwargs)
|
||||
|
||||
def __str__(self):
|
||||
return f"Execution of {self.flow.name} [{self.status}]"
|
||||
|
|
|
|||
39
workflows/tasks.py
Normal file
39
workflows/tasks.py
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
import logging
|
||||
from .models import FlowDefinition, FlowExecutionLog
|
||||
from .dispatcher import dispatch_flow
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def run_scheduled_flow(flow_id, window):
|
||||
"""
|
||||
Executes a scheduled flow idempotently for a given time window.
|
||||
If an execution already exists for this (flow_definition, window), skips execution.
|
||||
"""
|
||||
flow = FlowDefinition.objects.filter(id=flow_id).first()
|
||||
if not flow:
|
||||
logger.error(f"FlowDefinition {flow_id} not found.")
|
||||
return None
|
||||
|
||||
idempotency_key = f"scheduled:{flow_id}:{window}"
|
||||
|
||||
# Check if already executed or pending for this window
|
||||
existing = FlowExecutionLog.objects.filter(
|
||||
flow_definition=flow,
|
||||
idempotency_key=idempotency_key
|
||||
).first()
|
||||
if not existing:
|
||||
existing = FlowExecutionLog.objects.filter(
|
||||
flow=flow,
|
||||
idempotency_key=idempotency_key
|
||||
).first()
|
||||
|
||||
if existing:
|
||||
logger.info(f"Scheduled flow {flow_id} already executed for window {window}.")
|
||||
return existing
|
||||
|
||||
context = {
|
||||
'scheduled_window': window,
|
||||
'trigger': 'scheduled'
|
||||
}
|
||||
|
||||
return dispatch_flow(flow, context=context, idempotency_key=idempotency_key)
|
||||
Loading…
Reference in a new issue