feat(workflows): implement Sprint 4.2 advanced DAG nodes (Decision, Loop, Subflow) and Sprint 4.3 triggers (signals, reentrancy guard, celery idempotency)
This commit is contained in:
parent
d2771a2368
commit
5f2213ce09
7 changed files with 374 additions and 27 deletions
|
|
@ -76,7 +76,8 @@ class TestRecordTriggeredFlows:
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
class TestSignalReentrancyGuard:
|
class TestSignalReentrancyGuard:
|
||||||
|
|
||||||
def test_flow_updating_record_does_not_retrigger(self, db):
|
@pytest.mark.django_db(transaction=True)
|
||||||
|
def test_flow_updating_record_does_not_retrigger(self):
|
||||||
"""A flow that updates the triggering record must not cause infinite signal recursion."""
|
"""A flow that updates the triggering record must not cause infinite signal recursion."""
|
||||||
from workflows.models import FlowDefinition
|
from workflows.models import FlowDefinition
|
||||||
from crm_core.models import Account
|
from crm_core.models import Account
|
||||||
|
|
|
||||||
|
|
@ -3,3 +3,6 @@ from django.apps import AppConfig
|
||||||
class WorkflowsConfig(AppConfig):
|
class WorkflowsConfig(AppConfig):
|
||||||
default_auto_field = 'django.db.models.BigAutoField'
|
default_auto_field = 'django.db.models.BigAutoField'
|
||||||
name = 'workflows'
|
name = 'workflows'
|
||||||
|
|
||||||
|
def ready(self):
|
||||||
|
import workflows.dispatcher
|
||||||
|
|
|
||||||
112
workflows/dispatcher.py
Normal file
112
workflows/dispatcher.py
Normal file
|
|
@ -0,0 +1,112 @@
|
||||||
|
import threading
|
||||||
|
import logging
|
||||||
|
from django.db.models.signals import pre_save, post_save
|
||||||
|
from django.dispatch import receiver
|
||||||
|
from .models import FlowDefinition, FlowExecutionLog
|
||||||
|
from .engine import execute_flow_dag
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Thread-local storage for re-entrancy tracking and prior record state
|
||||||
|
_thread_locals = threading.local()
|
||||||
|
|
||||||
|
def _get_active_flow_depth():
|
||||||
|
return getattr(_thread_locals, 'active_flow_depth', 0)
|
||||||
|
|
||||||
|
def _set_active_flow_depth(val):
|
||||||
|
_thread_locals.active_flow_depth = val
|
||||||
|
|
||||||
|
MAX_SIGNAL_REENTRANCY_DEPTH = 1
|
||||||
|
|
||||||
|
def dispatch_flow(flow, context=None, idempotency_key=None):
|
||||||
|
"""
|
||||||
|
Dispatches a single flow execution with the given context.
|
||||||
|
"""
|
||||||
|
if context is None:
|
||||||
|
context = {}
|
||||||
|
return execute_flow_dag(flow, initial_context=context, idempotency_key=idempotency_key)
|
||||||
|
|
||||||
|
def snapshot_model(instance):
|
||||||
|
data = {}
|
||||||
|
if hasattr(instance, '__dict__'):
|
||||||
|
for k, v in instance.__dict__.items():
|
||||||
|
if not k.startswith('_') and isinstance(v, (str, int, float, bool, dict, list)):
|
||||||
|
data[k] = v
|
||||||
|
if 'id' not in data and getattr(instance, 'id', None):
|
||||||
|
data['id'] = instance.id
|
||||||
|
if 'name' not in data and hasattr(instance, 'name'):
|
||||||
|
data['name'] = instance.name
|
||||||
|
return data
|
||||||
|
|
||||||
|
@receiver(pre_save)
|
||||||
|
def on_record_pre_save(sender, instance, **kwargs):
|
||||||
|
app_label = sender._meta.app_label
|
||||||
|
if app_label not in ('crm_core', 'custom_objects'):
|
||||||
|
return
|
||||||
|
if sender.__name__ in ('FlowDefinition', 'FlowExecutionLog'):
|
||||||
|
return
|
||||||
|
|
||||||
|
if instance.pk:
|
||||||
|
try:
|
||||||
|
old_inst = sender.objects.filter(pk=instance.pk).first()
|
||||||
|
if old_inst:
|
||||||
|
instance._prior_state = snapshot_model(old_inst)
|
||||||
|
else:
|
||||||
|
instance._prior_state = {}
|
||||||
|
except Exception:
|
||||||
|
instance._prior_state = {}
|
||||||
|
else:
|
||||||
|
instance._prior_state = {}
|
||||||
|
|
||||||
|
@receiver(post_save)
|
||||||
|
def on_record_post_save(sender, instance, created, **kwargs):
|
||||||
|
app_label = sender._meta.app_label
|
||||||
|
if app_label not in ('crm_core', 'custom_objects'):
|
||||||
|
return
|
||||||
|
if sender.__name__ in ('FlowDefinition', 'FlowExecutionLog'):
|
||||||
|
return
|
||||||
|
|
||||||
|
depth = _get_active_flow_depth()
|
||||||
|
if depth >= MAX_SIGNAL_REENTRANCY_DEPTH:
|
||||||
|
logger.warning(
|
||||||
|
f"Re-entrancy guard blocked flow dispatch for {instance.__class__.__name__} "
|
||||||
|
f"(depth {depth} >= {MAX_SIGNAL_REENTRANCY_DEPTH})"
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
trigger_type = 'record_created' if created else 'record_updated'
|
||||||
|
model_name = sender.__name__
|
||||||
|
|
||||||
|
flows = FlowDefinition.objects.filter(
|
||||||
|
is_active=True,
|
||||||
|
trigger_type=trigger_type,
|
||||||
|
trigger_object=model_name
|
||||||
|
)
|
||||||
|
|
||||||
|
if not flows.exists():
|
||||||
|
flows = FlowDefinition.objects.filter(
|
||||||
|
is_active=True,
|
||||||
|
trigger_type=trigger_type,
|
||||||
|
trigger_object__iexact=model_name
|
||||||
|
)
|
||||||
|
|
||||||
|
current_state = snapshot_model(instance)
|
||||||
|
prior_state = getattr(instance, '_prior_state', {})
|
||||||
|
|
||||||
|
context = {
|
||||||
|
'$Record': current_state,
|
||||||
|
'$Record__Prior': prior_state,
|
||||||
|
'record_id': instance.pk,
|
||||||
|
'record_type': model_name,
|
||||||
|
'trigger': trigger_type
|
||||||
|
}
|
||||||
|
|
||||||
|
_set_active_flow_depth(depth + 1)
|
||||||
|
try:
|
||||||
|
for flow in flows:
|
||||||
|
try:
|
||||||
|
dispatch_flow(flow, context=context)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error executing flow {flow.id} on {trigger_type}: {e}")
|
||||||
|
finally:
|
||||||
|
_set_active_flow_depth(depth)
|
||||||
|
|
@ -1,7 +1,6 @@
|
||||||
from collections import deque
|
from collections import deque
|
||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
from django.core.exceptions import ValidationError
|
from django.core.exceptions import ValidationError
|
||||||
from .models import FlowExecutionLog
|
|
||||||
|
|
||||||
|
|
||||||
def validate_flow_dag(graph):
|
def validate_flow_dag(graph):
|
||||||
|
|
@ -88,30 +87,110 @@ def topological_sort(graph):
|
||||||
get_topological_execution_order = topological_sort
|
get_topological_execution_order = topological_sort
|
||||||
|
|
||||||
|
|
||||||
def execute_flow_dag(flow_def, initial_context=None, idempotency_key=None):
|
def evaluate_condition(cond, context):
|
||||||
"""
|
field = cond.get('field')
|
||||||
Executes a FlowDefinition DAG given context data.
|
op = cond.get('operator')
|
||||||
"""
|
target_val = cond.get('value')
|
||||||
if idempotency_key:
|
actual_val = context.get(field)
|
||||||
|
|
||||||
|
if op in ('gte', '>='):
|
||||||
|
return actual_val is not None and actual_val >= target_val
|
||||||
|
elif op in ('lte', '<='):
|
||||||
|
return actual_val is not None and actual_val <= target_val
|
||||||
|
elif op in ('gt', '>'):
|
||||||
|
return actual_val is not None and actual_val > target_val
|
||||||
|
elif op in ('lt', '<'):
|
||||||
|
return actual_val is not None and actual_val < target_val
|
||||||
|
elif op in ('eq', 'equals', '=='):
|
||||||
|
return actual_val == target_val
|
||||||
|
elif op in ('ne', 'not_equals', '!='):
|
||||||
|
return actual_val != target_val
|
||||||
|
elif op in ('contains',):
|
||||||
|
return target_val in str(actual_val or '')
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate_outcome(outcome, context):
|
||||||
|
conditions = outcome.get('conditions', [])
|
||||||
|
if not conditions:
|
||||||
|
return True # Default / Else branch
|
||||||
|
logic = outcome.get('logic', 'AND').upper()
|
||||||
|
results = [evaluate_condition(c, context) for c in conditions]
|
||||||
|
if logic == 'AND':
|
||||||
|
return all(results)
|
||||||
|
elif logic == 'OR':
|
||||||
|
return any(results)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def execute_flow_dag(flow_or_graph, initial_context=None, idempotency_key=None, depth=0):
|
||||||
|
from .models import FlowDefinition, FlowExecutionLog
|
||||||
|
|
||||||
|
MAX_SUBFLOW_DEPTH = 10
|
||||||
|
if depth >= MAX_SUBFLOW_DEPTH:
|
||||||
|
raise ValidationError(f"Subflow recursion depth limit exceeded (max: {MAX_SUBFLOW_DEPTH}).")
|
||||||
|
|
||||||
|
is_dict = isinstance(flow_or_graph, dict)
|
||||||
|
graph = flow_or_graph if is_dict else flow_or_graph.graph
|
||||||
|
flow_def = None if is_dict else flow_or_graph
|
||||||
|
|
||||||
|
validate_flow_dag(graph)
|
||||||
|
|
||||||
|
if not is_dict and idempotency_key:
|
||||||
existing = FlowExecutionLog.objects.filter(idempotency_key=idempotency_key).first()
|
existing = FlowExecutionLog.objects.filter(idempotency_key=idempotency_key).first()
|
||||||
if existing:
|
if existing:
|
||||||
return existing
|
return existing
|
||||||
|
|
||||||
log = FlowExecutionLog.objects.create(
|
log = None
|
||||||
flow=flow_def,
|
if not is_dict and flow_def:
|
||||||
status='running',
|
log = FlowExecutionLog.objects.create(
|
||||||
context_data=initial_context or {},
|
flow=flow_def,
|
||||||
idempotency_key=idempotency_key
|
status='running',
|
||||||
)
|
context_data=initial_context or {},
|
||||||
|
idempotency_key=idempotency_key
|
||||||
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
execution_order = topological_sort(flow_def.graph)
|
nodes = graph.get('nodes', [])
|
||||||
node_map = {n['id']: n for n in flow_def.graph.get('nodes', [])}
|
edges = graph.get('edges', [])
|
||||||
|
node_map = {n['id']: n for n in nodes if 'id' in n}
|
||||||
|
|
||||||
|
# Build outgoing edge maps
|
||||||
|
outgoing_edges = {}
|
||||||
|
for edge in edges:
|
||||||
|
src = edge.get('source')
|
||||||
|
outgoing_edges.setdefault(src, []).append(edge)
|
||||||
|
|
||||||
|
# Topological sorting
|
||||||
|
in_degree = {nid: 0 for nid in node_map}
|
||||||
|
adj = {nid: [] for nid in node_map}
|
||||||
|
for edge in edges:
|
||||||
|
src = edge.get('source')
|
||||||
|
tgt = edge.get('target')
|
||||||
|
adj[src].append(tgt)
|
||||||
|
in_degree[tgt] += 1
|
||||||
|
|
||||||
|
queue = deque([nid for nid in node_map if in_degree[nid] == 0])
|
||||||
|
topo_order = []
|
||||||
|
while queue:
|
||||||
|
curr = queue.popleft()
|
||||||
|
topo_order.append(curr)
|
||||||
|
for neighbor in adj[curr]:
|
||||||
|
in_degree[neighbor] -= 1
|
||||||
|
if in_degree[neighbor] == 0:
|
||||||
|
queue.append(neighbor)
|
||||||
|
|
||||||
|
# Determine reachable nodes taking decision branches into account
|
||||||
|
reachable_nodes = set(topo_order)
|
||||||
|
disabled_nodes = set()
|
||||||
|
|
||||||
steps = []
|
|
||||||
context = dict(initial_context or {})
|
context = dict(initial_context or {})
|
||||||
|
steps = []
|
||||||
|
|
||||||
|
for node_id in topo_order:
|
||||||
|
if node_id in disabled_nodes:
|
||||||
|
continue
|
||||||
|
|
||||||
for node_id in execution_order:
|
|
||||||
node = node_map.get(node_id, {})
|
node = node_map.get(node_id, {})
|
||||||
node_type = node.get('type')
|
node_type = node.get('type')
|
||||||
node_data = node.get('data', {})
|
node_data = node.get('data', {})
|
||||||
|
|
@ -134,18 +213,98 @@ def execute_flow_dag(flow_def, initial_context=None, idempotency_key=None):
|
||||||
pts = node_data.get('points', 0)
|
pts = node_data.get('points', 0)
|
||||||
context['score'] = context.get('score', 0) + pts
|
context['score'] = context.get('score', 0) + pts
|
||||||
|
|
||||||
|
elif node_type == 'update_record':
|
||||||
|
for k, v in node_data.items():
|
||||||
|
context[k] = v
|
||||||
|
# Check if this flow was triggered by a model record and update DB record if field/value specified
|
||||||
|
field = node_data.get('field')
|
||||||
|
val = node_data.get('value')
|
||||||
|
record_id = context.get('record_id')
|
||||||
|
record_type = context.get('record_type')
|
||||||
|
if field and val is not None and record_id and record_type:
|
||||||
|
from django.apps import apps
|
||||||
|
model_cls = apps.get_model('crm_core', record_type)
|
||||||
|
if not model_cls:
|
||||||
|
model_cls = apps.get_model('custom_objects', record_type)
|
||||||
|
if model_cls:
|
||||||
|
inst = model_cls.objects.filter(pk=record_id).first()
|
||||||
|
if inst and hasattr(inst, field):
|
||||||
|
setattr(inst, field, val)
|
||||||
|
inst.save()
|
||||||
|
|
||||||
|
elif node_type == 'assignment':
|
||||||
|
var_name = node_data.get('variable')
|
||||||
|
op = node_data.get('operator', 'equals')
|
||||||
|
val = node_data.get('value')
|
||||||
|
if var_name:
|
||||||
|
if op in ('equals', '=', None):
|
||||||
|
context[var_name] = val
|
||||||
|
elif op in ('add', '+='):
|
||||||
|
curr = context.get(var_name, 0)
|
||||||
|
context[var_name] = curr + val
|
||||||
|
elif op in ('subtract', '-='):
|
||||||
|
curr = context.get(var_name, 0)
|
||||||
|
context[var_name] = curr - val
|
||||||
|
|
||||||
|
elif node_type == 'decision':
|
||||||
|
outcomes = node_data.get('outcomes', [])
|
||||||
|
winning_handle = None
|
||||||
|
for outcome in outcomes:
|
||||||
|
if evaluate_outcome(outcome, context):
|
||||||
|
winning_handle = outcome.get('id')
|
||||||
|
break
|
||||||
|
|
||||||
|
# Disable paths from unchosen handles
|
||||||
|
node_edges = outgoing_edges.get(node_id, [])
|
||||||
|
for e in node_edges:
|
||||||
|
handle = e.get('sourceHandle')
|
||||||
|
target = e.get('target')
|
||||||
|
if handle != winning_handle:
|
||||||
|
# Disable target and all downstream exclusively reached nodes
|
||||||
|
downstream = [target]
|
||||||
|
visited_sub = set()
|
||||||
|
while downstream:
|
||||||
|
d = downstream.pop()
|
||||||
|
if d not in visited_sub:
|
||||||
|
visited_sub.add(d)
|
||||||
|
disabled_nodes.add(d)
|
||||||
|
for nxt_edge in outgoing_edges.get(d, []):
|
||||||
|
downstream.append(nxt_edge.get('target'))
|
||||||
|
|
||||||
|
elif node_type == 'subflow':
|
||||||
|
child_flow_id = node_data.get('flow_id')
|
||||||
|
if not child_flow_id:
|
||||||
|
raise ValidationError("Subflow node requires flow_id.")
|
||||||
|
child_flow = FlowDefinition.objects.filter(pk=child_flow_id).first()
|
||||||
|
if not child_flow:
|
||||||
|
raise ValidationError(f"Referenced child flow {child_flow_id} not found.")
|
||||||
|
|
||||||
|
child_res = execute_flow_dag(
|
||||||
|
child_flow.graph,
|
||||||
|
initial_context=context,
|
||||||
|
depth=depth + 1
|
||||||
|
)
|
||||||
|
if isinstance(child_res, dict):
|
||||||
|
context.update(child_res)
|
||||||
|
elif hasattr(child_res, 'context_data'):
|
||||||
|
context.update(child_res.context_data)
|
||||||
|
|
||||||
steps.append(step_record)
|
steps.append(step_record)
|
||||||
|
|
||||||
log.status = 'completed'
|
if log:
|
||||||
log.steps_executed = steps
|
log.status = 'completed'
|
||||||
log.context_data = context
|
log.steps_executed = steps
|
||||||
log.completed_at = timezone.now()
|
log.context_data = context
|
||||||
log.save()
|
log.completed_at = timezone.now()
|
||||||
return log
|
log.save()
|
||||||
|
return log
|
||||||
|
|
||||||
|
return context
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.status = 'failed'
|
if log:
|
||||||
log.error_message = str(e)
|
log.status = 'failed'
|
||||||
log.completed_at = timezone.now()
|
log.error_message = str(e)
|
||||||
log.save()
|
log.completed_at = timezone.now()
|
||||||
|
log.save()
|
||||||
raise
|
raise
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,24 @@
|
||||||
|
# Generated by Django 4.2.30 on 2026-09-26 17:36
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
import django.db.models.deletion
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('workflows', '0001_initial'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AddField(
|
||||||
|
model_name='flowdefinition',
|
||||||
|
name='trigger_object',
|
||||||
|
field=models.CharField(blank=True, max_length=100, null=True),
|
||||||
|
),
|
||||||
|
migrations.AddField(
|
||||||
|
model_name='flowexecutionlog',
|
||||||
|
name='flow_definition',
|
||||||
|
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='legacy_executions', to='workflows.flowdefinition'),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
@ -16,6 +16,7 @@ class FlowDefinition(models.Model):
|
||||||
name = models.CharField(max_length=255)
|
name = models.CharField(max_length=255)
|
||||||
description = models.TextField(blank=True, default='')
|
description = models.TextField(blank=True, default='')
|
||||||
trigger_type = models.CharField(max_length=50, choices=TRIGGER_TYPE_CHOICES, default='manual')
|
trigger_type = models.CharField(max_length=50, choices=TRIGGER_TYPE_CHOICES, default='manual')
|
||||||
|
trigger_object = models.CharField(max_length=100, blank=True, null=True)
|
||||||
is_active = models.BooleanField(default=True)
|
is_active = models.BooleanField(default=True)
|
||||||
|
|
||||||
# Vue Flow Graph: { "nodes": [...], "edges": [...] }
|
# Vue Flow Graph: { "nodes": [...], "edges": [...] }
|
||||||
|
|
@ -53,6 +54,7 @@ class FlowExecutionLog(models.Model):
|
||||||
]
|
]
|
||||||
|
|
||||||
flow = models.ForeignKey(FlowDefinition, on_delete=models.CASCADE, related_name='executions')
|
flow = models.ForeignKey(FlowDefinition, on_delete=models.CASCADE, related_name='executions')
|
||||||
|
flow_definition = models.ForeignKey(FlowDefinition, null=True, blank=True, on_delete=models.CASCADE, related_name='legacy_executions')
|
||||||
status = models.CharField(max_length=20, choices=STATUS_CHOICES, default='pending')
|
status = models.CharField(max_length=20, choices=STATUS_CHOICES, default='pending')
|
||||||
started_at = models.DateTimeField(auto_now_add=True)
|
started_at = models.DateTimeField(auto_now_add=True)
|
||||||
completed_at = models.DateTimeField(null=True, blank=True)
|
completed_at = models.DateTimeField(null=True, blank=True)
|
||||||
|
|
@ -79,5 +81,12 @@ class FlowExecutionLog(models.Model):
|
||||||
def context(self, value):
|
def context(self, value):
|
||||||
self.context_data = value
|
self.context_data = value
|
||||||
|
|
||||||
|
def save(self, *args, **kwargs):
|
||||||
|
if self.flow_id and not self.flow_definition_id:
|
||||||
|
self.flow_definition = self.flow
|
||||||
|
elif self.flow_definition_id and not self.flow_id:
|
||||||
|
self.flow = self.flow_definition
|
||||||
|
super().save(*args, **kwargs)
|
||||||
|
|
||||||
def __str__(self):
|
def __str__(self):
|
||||||
return f"Execution of {self.flow.name} [{self.status}]"
|
return f"Execution of {self.flow.name} [{self.status}]"
|
||||||
|
|
|
||||||
39
workflows/tasks.py
Normal file
39
workflows/tasks.py
Normal file
|
|
@ -0,0 +1,39 @@
|
||||||
|
import logging
|
||||||
|
from .models import FlowDefinition, FlowExecutionLog
|
||||||
|
from .dispatcher import dispatch_flow
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
def run_scheduled_flow(flow_id, window):
|
||||||
|
"""
|
||||||
|
Executes a scheduled flow idempotently for a given time window.
|
||||||
|
If an execution already exists for this (flow_definition, window), skips execution.
|
||||||
|
"""
|
||||||
|
flow = FlowDefinition.objects.filter(id=flow_id).first()
|
||||||
|
if not flow:
|
||||||
|
logger.error(f"FlowDefinition {flow_id} not found.")
|
||||||
|
return None
|
||||||
|
|
||||||
|
idempotency_key = f"scheduled:{flow_id}:{window}"
|
||||||
|
|
||||||
|
# Check if already executed or pending for this window
|
||||||
|
existing = FlowExecutionLog.objects.filter(
|
||||||
|
flow_definition=flow,
|
||||||
|
idempotency_key=idempotency_key
|
||||||
|
).first()
|
||||||
|
if not existing:
|
||||||
|
existing = FlowExecutionLog.objects.filter(
|
||||||
|
flow=flow,
|
||||||
|
idempotency_key=idempotency_key
|
||||||
|
).first()
|
||||||
|
|
||||||
|
if existing:
|
||||||
|
logger.info(f"Scheduled flow {flow_id} already executed for window {window}.")
|
||||||
|
return existing
|
||||||
|
|
||||||
|
context = {
|
||||||
|
'scheduled_window': window,
|
||||||
|
'trigger': 'scheduled'
|
||||||
|
}
|
||||||
|
|
||||||
|
return dispatch_flow(flow, context=context, idempotency_key=idempotency_key)
|
||||||
Loading…
Reference in a new issue