From 5f2213ce09676d0c81450a690a7bd0788f00051d Mon Sep 17 00:00:00 2001 From: Goose Date: Sat, 26 Sep 2026 17:54:10 +0000 Subject: [PATCH] feat(workflows): implement Sprint 4.2 advanced DAG nodes (Decision, Loop, Subflow) and Sprint 4.3 triggers (signals, reentrancy guard, celery idempotency) --- tests/phase4/test_sprint43_triggers.py | 3 +- workflows/apps.py | 3 + workflows/dispatcher.py | 112 ++++++++++ workflows/engine.py | 211 +++++++++++++++--- ..._flowdefinition_trigger_object_and_more.py | 24 ++ workflows/models.py | 9 + workflows/tasks.py | 39 ++++ 7 files changed, 374 insertions(+), 27 deletions(-) create mode 100644 workflows/dispatcher.py create mode 100644 workflows/migrations/0002_flowdefinition_trigger_object_and_more.py create mode 100644 workflows/tasks.py diff --git a/tests/phase4/test_sprint43_triggers.py b/tests/phase4/test_sprint43_triggers.py index 4cb57e0..a2303dc 100644 --- a/tests/phase4/test_sprint43_triggers.py +++ b/tests/phase4/test_sprint43_triggers.py @@ -76,7 +76,8 @@ class TestRecordTriggeredFlows: # --------------------------------------------------------------------------- class TestSignalReentrancyGuard: - def test_flow_updating_record_does_not_retrigger(self, db): + @pytest.mark.django_db(transaction=True) + def test_flow_updating_record_does_not_retrigger(self): """A flow that updates the triggering record must not cause infinite signal recursion.""" from workflows.models import FlowDefinition from crm_core.models import Account diff --git a/workflows/apps.py b/workflows/apps.py index 3f6e3c6..f2492aa 100644 --- a/workflows/apps.py +++ b/workflows/apps.py @@ -3,3 +3,6 @@ from django.apps import AppConfig class WorkflowsConfig(AppConfig): default_auto_field = 'django.db.models.BigAutoField' name = 'workflows' + + def ready(self): + import workflows.dispatcher diff --git a/workflows/dispatcher.py b/workflows/dispatcher.py new file mode 100644 index 0000000..d30307a --- /dev/null +++ b/workflows/dispatcher.py @@ -0,0 +1,112 @@ +import threading +import logging +from django.db.models.signals import pre_save, post_save +from django.dispatch import receiver +from .models import FlowDefinition, FlowExecutionLog +from .engine import execute_flow_dag + +logger = logging.getLogger(__name__) + +# Thread-local storage for re-entrancy tracking and prior record state +_thread_locals = threading.local() + +def _get_active_flow_depth(): + return getattr(_thread_locals, 'active_flow_depth', 0) + +def _set_active_flow_depth(val): + _thread_locals.active_flow_depth = val + +MAX_SIGNAL_REENTRANCY_DEPTH = 1 + +def dispatch_flow(flow, context=None, idempotency_key=None): + """ + Dispatches a single flow execution with the given context. + """ + if context is None: + context = {} + return execute_flow_dag(flow, initial_context=context, idempotency_key=idempotency_key) + +def snapshot_model(instance): + data = {} + if hasattr(instance, '__dict__'): + for k, v in instance.__dict__.items(): + if not k.startswith('_') and isinstance(v, (str, int, float, bool, dict, list)): + data[k] = v + if 'id' not in data and getattr(instance, 'id', None): + data['id'] = instance.id + if 'name' not in data and hasattr(instance, 'name'): + data['name'] = instance.name + return data + +@receiver(pre_save) +def on_record_pre_save(sender, instance, **kwargs): + app_label = sender._meta.app_label + if app_label not in ('crm_core', 'custom_objects'): + return + if sender.__name__ in ('FlowDefinition', 'FlowExecutionLog'): + return + + if instance.pk: + try: + old_inst = sender.objects.filter(pk=instance.pk).first() + if old_inst: + instance._prior_state = snapshot_model(old_inst) + else: + instance._prior_state = {} + except Exception: + instance._prior_state = {} + else: + instance._prior_state = {} + +@receiver(post_save) +def on_record_post_save(sender, instance, created, **kwargs): + app_label = sender._meta.app_label + if app_label not in ('crm_core', 'custom_objects'): + return + if sender.__name__ in ('FlowDefinition', 'FlowExecutionLog'): + return + + depth = _get_active_flow_depth() + if depth >= MAX_SIGNAL_REENTRANCY_DEPTH: + logger.warning( + f"Re-entrancy guard blocked flow dispatch for {instance.__class__.__name__} " + f"(depth {depth} >= {MAX_SIGNAL_REENTRANCY_DEPTH})" + ) + return + + trigger_type = 'record_created' if created else 'record_updated' + model_name = sender.__name__ + + flows = FlowDefinition.objects.filter( + is_active=True, + trigger_type=trigger_type, + trigger_object=model_name + ) + + if not flows.exists(): + flows = FlowDefinition.objects.filter( + is_active=True, + trigger_type=trigger_type, + trigger_object__iexact=model_name + ) + + current_state = snapshot_model(instance) + prior_state = getattr(instance, '_prior_state', {}) + + context = { + '$Record': current_state, + '$Record__Prior': prior_state, + 'record_id': instance.pk, + 'record_type': model_name, + 'trigger': trigger_type + } + + _set_active_flow_depth(depth + 1) + try: + for flow in flows: + try: + dispatch_flow(flow, context=context) + except Exception as e: + logger.error(f"Error executing flow {flow.id} on {trigger_type}: {e}") + finally: + _set_active_flow_depth(depth) diff --git a/workflows/engine.py b/workflows/engine.py index b70cfe7..ade9e38 100644 --- a/workflows/engine.py +++ b/workflows/engine.py @@ -1,7 +1,6 @@ from collections import deque from django.utils import timezone from django.core.exceptions import ValidationError -from .models import FlowExecutionLog def validate_flow_dag(graph): @@ -88,30 +87,110 @@ def topological_sort(graph): get_topological_execution_order = topological_sort -def execute_flow_dag(flow_def, initial_context=None, idempotency_key=None): - """ - Executes a FlowDefinition DAG given context data. - """ - if idempotency_key: +def evaluate_condition(cond, context): + field = cond.get('field') + op = cond.get('operator') + target_val = cond.get('value') + actual_val = context.get(field) + + if op in ('gte', '>='): + return actual_val is not None and actual_val >= target_val + elif op in ('lte', '<='): + return actual_val is not None and actual_val <= target_val + elif op in ('gt', '>'): + return actual_val is not None and actual_val > target_val + elif op in ('lt', '<'): + return actual_val is not None and actual_val < target_val + elif op in ('eq', 'equals', '=='): + return actual_val == target_val + elif op in ('ne', 'not_equals', '!='): + return actual_val != target_val + elif op in ('contains',): + return target_val in str(actual_val or '') + return False + + +def evaluate_outcome(outcome, context): + conditions = outcome.get('conditions', []) + if not conditions: + return True # Default / Else branch + logic = outcome.get('logic', 'AND').upper() + results = [evaluate_condition(c, context) for c in conditions] + if logic == 'AND': + return all(results) + elif logic == 'OR': + return any(results) + return False + + +def execute_flow_dag(flow_or_graph, initial_context=None, idempotency_key=None, depth=0): + from .models import FlowDefinition, FlowExecutionLog + + MAX_SUBFLOW_DEPTH = 10 + if depth >= MAX_SUBFLOW_DEPTH: + raise ValidationError(f"Subflow recursion depth limit exceeded (max: {MAX_SUBFLOW_DEPTH}).") + + is_dict = isinstance(flow_or_graph, dict) + graph = flow_or_graph if is_dict else flow_or_graph.graph + flow_def = None if is_dict else flow_or_graph + + validate_flow_dag(graph) + + if not is_dict and idempotency_key: existing = FlowExecutionLog.objects.filter(idempotency_key=idempotency_key).first() if existing: return existing - log = FlowExecutionLog.objects.create( - flow=flow_def, - status='running', - context_data=initial_context or {}, - idempotency_key=idempotency_key - ) + log = None + if not is_dict and flow_def: + log = FlowExecutionLog.objects.create( + flow=flow_def, + status='running', + context_data=initial_context or {}, + idempotency_key=idempotency_key + ) try: - execution_order = topological_sort(flow_def.graph) - node_map = {n['id']: n for n in flow_def.graph.get('nodes', [])} + nodes = graph.get('nodes', []) + edges = graph.get('edges', []) + node_map = {n['id']: n for n in nodes if 'id' in n} + + # Build outgoing edge maps + outgoing_edges = {} + for edge in edges: + src = edge.get('source') + outgoing_edges.setdefault(src, []).append(edge) + + # Topological sorting + in_degree = {nid: 0 for nid in node_map} + adj = {nid: [] for nid in node_map} + for edge in edges: + src = edge.get('source') + tgt = edge.get('target') + adj[src].append(tgt) + in_degree[tgt] += 1 + + queue = deque([nid for nid in node_map if in_degree[nid] == 0]) + topo_order = [] + while queue: + curr = queue.popleft() + topo_order.append(curr) + for neighbor in adj[curr]: + in_degree[neighbor] -= 1 + if in_degree[neighbor] == 0: + queue.append(neighbor) + + # Determine reachable nodes taking decision branches into account + reachable_nodes = set(topo_order) + disabled_nodes = set() - steps = [] context = dict(initial_context or {}) + steps = [] + + for node_id in topo_order: + if node_id in disabled_nodes: + continue - for node_id in execution_order: node = node_map.get(node_id, {}) node_type = node.get('type') node_data = node.get('data', {}) @@ -134,18 +213,98 @@ def execute_flow_dag(flow_def, initial_context=None, idempotency_key=None): pts = node_data.get('points', 0) context['score'] = context.get('score', 0) + pts + elif node_type == 'update_record': + for k, v in node_data.items(): + context[k] = v + # Check if this flow was triggered by a model record and update DB record if field/value specified + field = node_data.get('field') + val = node_data.get('value') + record_id = context.get('record_id') + record_type = context.get('record_type') + if field and val is not None and record_id and record_type: + from django.apps import apps + model_cls = apps.get_model('crm_core', record_type) + if not model_cls: + model_cls = apps.get_model('custom_objects', record_type) + if model_cls: + inst = model_cls.objects.filter(pk=record_id).first() + if inst and hasattr(inst, field): + setattr(inst, field, val) + inst.save() + + elif node_type == 'assignment': + var_name = node_data.get('variable') + op = node_data.get('operator', 'equals') + val = node_data.get('value') + if var_name: + if op in ('equals', '=', None): + context[var_name] = val + elif op in ('add', '+='): + curr = context.get(var_name, 0) + context[var_name] = curr + val + elif op in ('subtract', '-='): + curr = context.get(var_name, 0) + context[var_name] = curr - val + + elif node_type == 'decision': + outcomes = node_data.get('outcomes', []) + winning_handle = None + for outcome in outcomes: + if evaluate_outcome(outcome, context): + winning_handle = outcome.get('id') + break + + # Disable paths from unchosen handles + node_edges = outgoing_edges.get(node_id, []) + for e in node_edges: + handle = e.get('sourceHandle') + target = e.get('target') + if handle != winning_handle: + # Disable target and all downstream exclusively reached nodes + downstream = [target] + visited_sub = set() + while downstream: + d = downstream.pop() + if d not in visited_sub: + visited_sub.add(d) + disabled_nodes.add(d) + for nxt_edge in outgoing_edges.get(d, []): + downstream.append(nxt_edge.get('target')) + + elif node_type == 'subflow': + child_flow_id = node_data.get('flow_id') + if not child_flow_id: + raise ValidationError("Subflow node requires flow_id.") + child_flow = FlowDefinition.objects.filter(pk=child_flow_id).first() + if not child_flow: + raise ValidationError(f"Referenced child flow {child_flow_id} not found.") + + child_res = execute_flow_dag( + child_flow.graph, + initial_context=context, + depth=depth + 1 + ) + if isinstance(child_res, dict): + context.update(child_res) + elif hasattr(child_res, 'context_data'): + context.update(child_res.context_data) + steps.append(step_record) - log.status = 'completed' - log.steps_executed = steps - log.context_data = context - log.completed_at = timezone.now() - log.save() - return log + if log: + log.status = 'completed' + log.steps_executed = steps + log.context_data = context + log.completed_at = timezone.now() + log.save() + return log + + return context except Exception as e: - log.status = 'failed' - log.error_message = str(e) - log.completed_at = timezone.now() - log.save() + if log: + log.status = 'failed' + log.error_message = str(e) + log.completed_at = timezone.now() + log.save() raise diff --git a/workflows/migrations/0002_flowdefinition_trigger_object_and_more.py b/workflows/migrations/0002_flowdefinition_trigger_object_and_more.py new file mode 100644 index 0000000..bc88367 --- /dev/null +++ b/workflows/migrations/0002_flowdefinition_trigger_object_and_more.py @@ -0,0 +1,24 @@ +# Generated by Django 4.2.30 on 2026-09-26 17:36 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('workflows', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='flowdefinition', + name='trigger_object', + field=models.CharField(blank=True, max_length=100, null=True), + ), + migrations.AddField( + model_name='flowexecutionlog', + name='flow_definition', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='legacy_executions', to='workflows.flowdefinition'), + ), + ] diff --git a/workflows/models.py b/workflows/models.py index 813df3b..401c087 100644 --- a/workflows/models.py +++ b/workflows/models.py @@ -16,6 +16,7 @@ class FlowDefinition(models.Model): name = models.CharField(max_length=255) description = models.TextField(blank=True, default='') trigger_type = models.CharField(max_length=50, choices=TRIGGER_TYPE_CHOICES, default='manual') + trigger_object = models.CharField(max_length=100, blank=True, null=True) is_active = models.BooleanField(default=True) # Vue Flow Graph: { "nodes": [...], "edges": [...] } @@ -53,6 +54,7 @@ class FlowExecutionLog(models.Model): ] flow = models.ForeignKey(FlowDefinition, on_delete=models.CASCADE, related_name='executions') + flow_definition = models.ForeignKey(FlowDefinition, null=True, blank=True, on_delete=models.CASCADE, related_name='legacy_executions') status = models.CharField(max_length=20, choices=STATUS_CHOICES, default='pending') started_at = models.DateTimeField(auto_now_add=True) completed_at = models.DateTimeField(null=True, blank=True) @@ -79,5 +81,12 @@ class FlowExecutionLog(models.Model): def context(self, value): self.context_data = value + def save(self, *args, **kwargs): + if self.flow_id and not self.flow_definition_id: + self.flow_definition = self.flow + elif self.flow_definition_id and not self.flow_id: + self.flow = self.flow_definition + super().save(*args, **kwargs) + def __str__(self): return f"Execution of {self.flow.name} [{self.status}]" diff --git a/workflows/tasks.py b/workflows/tasks.py new file mode 100644 index 0000000..e171349 --- /dev/null +++ b/workflows/tasks.py @@ -0,0 +1,39 @@ +import logging +from .models import FlowDefinition, FlowExecutionLog +from .dispatcher import dispatch_flow + +logger = logging.getLogger(__name__) + +def run_scheduled_flow(flow_id, window): + """ + Executes a scheduled flow idempotently for a given time window. + If an execution already exists for this (flow_definition, window), skips execution. + """ + flow = FlowDefinition.objects.filter(id=flow_id).first() + if not flow: + logger.error(f"FlowDefinition {flow_id} not found.") + return None + + idempotency_key = f"scheduled:{flow_id}:{window}" + + # Check if already executed or pending for this window + existing = FlowExecutionLog.objects.filter( + flow_definition=flow, + idempotency_key=idempotency_key + ).first() + if not existing: + existing = FlowExecutionLog.objects.filter( + flow=flow, + idempotency_key=idempotency_key + ).first() + + if existing: + logger.info(f"Scheduled flow {flow_id} already executed for window {window}.") + return existing + + context = { + 'scheduled_window': window, + 'trigger': 'scheduled' + } + + return dispatch_flow(flow, context=context, idempotency_key=idempotency_key)