// What starts simple...
{
  "name": "Process_Customer_Data",
  "activities": [
    {"type": "Copy"},
    {"type": "DataFlow"},
    {"type": "StoredProcedure"}
  ]
}
// ...becomes unwieldy
{
  "name": "Process_All_Customer_Data",
  "activities": [
    // Activity 1-10: Validation
    // Activity 11-20: Extraction
    // Activity 21-30: Transformation
    // Activity 31-40: Loading
    // Activity 41: ERROR - Pipeline exceeds maximum activity limit
  ]
}

//Option 1: Split into orchestration pattern
{
  "name": "Orchestrator_Pipeline",
  "activities": [
    {"type": "ExecutePipeline", "pipeline": "Validate_Data"},
    {"type": "ExecutePipeline", "pipeline": "Extract_Data"},
    {"type": "ExecutePipeline", "pipeline": "Transform_Data"},
    {"type": "ExecutePipeline", "pipeline": "Load_Data"}
  ]
}
// Option 2: Move complex logic elsewhere (Databricks, Functions)
{
  "name": "Process_Data",
  "activities": [
    {"type": "DatabricksNotebook", "notebookPath": "/Pipelines/ActualLogic"}
  ]
}

# Scenario: Processing 100GB daily across 50 pipelines
# Azure Integration Runtime costs
data_movement_cost = 100  # GB per day
diu_hours = 50  # Pipeline executions * avg runtime
activity_runs = 500  # Activities per day
monthly_ir_cost = (
    (data_movement_cost * 30 * 0.25) +  # $0.25/GB
    (diu_hours * 30 * 0.25) +           # $0.25/DIU-hour
    (activity_runs * 30 * 0.001)        # $0.001/activity
)
# = $750/month base
# Actual bill: Often 3-5x higher due to:
# - Data movement between regions
# - Failed retries (you pay for failures)
# - Suboptimal DIU allocation
# - Debugging runs

# What you want (standard software engineering)
git checkout -b feature/new-customer-pipeline
# Make changes, test locally
pytest tests/
# Deploy via CI/CD
git push origin feature/new-customer-pipeline
# Automated: code review, tests, deployment
# What you get with ADF
# 1. Make changes in ADF UI (connected to Git branch)
# 2. Changes auto-committed to ARM templates (not human-readable)
# 3. No local development or testing
# 4. Manual deployment via Azure DevOps or GitHub Actions
# 5. No easy way to diff changes
# 6. Can't easily rollback (ARM template restore is painful)

// ADF saves pipelines as ARM templates
{
  "name": "Simple_Copy_Pipeline",
  "properties": {
    "activities": [
      {
        "name": "CopyData",
        "type": "Copy",
        "dependsOn": [],
        "policy": {
          "timeout": "7.00:00:00",
          "retry": 0,
          "retryIntervalInSeconds": 30,
          "secureOutput": false,
          "secureInput": false
        },
        "userProperties": [],
        "typeProperties": {
          // 200+ more lines of JSON...
        }
      }
    ]
  }
}
// Good luck code reviewing this

# 1. Get paged at 3 AM - pipeline failed
# 2. Log into Azure Portal
# 3. Navigate to Data Factory
# 4. Find failed pipeline run
# 5. Click through nested pipeline executions
# 6. Find failed activity (maybe 6 levels deep)
# 7. Read cryptic error message:
"Activity failed because of an unhandled exception"
# 8. No stack trace, no context, no input parameters logged
# 9. Re-run with debug mode (takes 20 minutes)
# 10. Fails differently because data changed
# 11. Repeat 3-4 times until you guess the issue
# Total time: 2 hours

# Databricks notebook automatically logs:
# - Cell execution time
# - Exception stack traces
# - Variable state at failure
# - Cluster metrics (CPU, memory, I/O)
# - Spark UI for query plans
# - Job parameters and configuration
# Total debug time: 10 minutes

# Custom logging wrapper every team creates
def log_activity_start(activity_name, parameters):
    log_analytics_client.log({
        "timestamp": datetime.now(),
        "pipeline": pipeline_name,
        "activity": activity_name,
        "parameters": parameters,
        "status": "started"
    })
def log_activity_end(activity_name, status, error=None):
    log_analytics_client.log({
        "timestamp": datetime.now(),
        "pipeline": pipeline_name,
        "activity": activity_name,
        "status": status,
        "error": str(error) if error else None
    })
# Now multiply this by every pipeline...

// In Databricks/Spark - natural
val processedOrders = orders
  .withColumn("total_with_tax",
    when(col("state") === "CA", col("amount") * 1.0725)
    .when(col("state") === "NY", col("amount") * 1.08875)
    .otherwise(col("amount") * 1.06)
  )
  .withColumn("risk_score", calculateRisk(col("customer_id"), col("amount")))
  .filter(col("risk_score") < 0.8)
// In ADF Data Flow - requires expression builder GUI
// Complex logic becomes unreadable
// No custom functions without workarounds
// Version control is a JSON nightmare

# Spark/Databricks: Full control
df.repartition(200, "customer_id")  # Optimize partitioning
df.cache()                          # Control caching
spark.conf.set("spark.sql.shuffle.partitions", "200")
# ADF Data Flow: Limited controls
# - Auto-optimization (black box)
# - Can't control partitioning strategy
# - Can't manually cache
# - Can't tune Spark configs
# Hope the optimizer guesses right

// Most teams end up doing this
{
  "name": "Process_Complex_Data",
  "activities": [
    {
      "name": "CopyToStaging",
      "type": "Copy"
    },
    {
      "name": "RunActualLogic",
      "type": "DatabricksNotebook",
      "notebook": "/Transformations/ComplexBusinessLogic"
    },
    {
      "name": "CopyResults",
      "type": "Copy"
    }
  ]
}
// ADF becomes a thin orchestration layer
// All real work happens in Databricks
// Question: Why pay for both?

# What you need
# Process data arriving every 15 minutes
# Handle late-arriving data (2-hour window)
# Automatically retry failures
# Track high-water marks
# What ADF gives you
{
  "type": "TumblingWindowTrigger",
  "frequency": "Minute",
  "interval": 15,
  "startTime": "2024-01-01T00:00:00Z",
  "delay": "02:00:00",  # Late arrival window
  "maxConcurrency": 10,  # But...
  # - Can't dynamically adjust based on data volume
  # - Limited backfill capabilities
  # - Retry logic is basic
  # - No concept of data dependencies
}

# Trigger on blob storage events
{
  "type": "BlobEventsTrigger",
  "events": ["Microsoft.Storage.BlobCreated"],
  "scope": "/subscriptions/.../blobServices/default/containers/data"
  # Problems:
  # - Can miss events if ADF is deploying
  # - No ordering guarantees
  # - Can't easily process multiple files as batch
  # - Retries create duplicate processing
}

# What you want (Airflow-style)
pipeline_a >> pipeline_b >> pipeline_c
pipeline_d >> [pipeline_b, pipeline_e]  # Multiple dependencies
# What you get in ADF
# - Can trigger pipeline B when pipeline A succeeds
# - Can't express "wait for A AND D before running B"
# - Workarounds involve checking blob markers or database flags
# - Gets messy fast

# Common need: Generate pipelines for 100+ data sources
# In Airflow:
for source in config.sources:
    dag.add_task(
        PythonOperator(
            task_id=f"process_{source.name}",
            python_callable=process_data,
            op_kwargs={"source": source}
        )
    )
# In ADF:
# - Must manually create 100 pipelines in UI
# - Or generate 100 ARM templates with scripts
# - Or use overly complex ForEach loops (with limits)

# What you want
def test_customer_transformation():
    input_df = pd.DataFrame({"id": [1, 2], "name": ["Alice", "Bob"]})
    result = transform_customers(input_df)
    assert result.shape[0] == 2
    assert "name_upper" in result.columns
# What you get with ADF
# - Can't unit test transformations
# - Can't mock data sources
# - Must deploy and run actual pipelines
# - Testing environment = copy of production ADF ($$$)

# Standard software engineering
from data_utils import validate_schema, clean_nulls, deduplicate
df = (
    raw_data
    .transform(validate_schema)
    .transform(clean_nulls)
    .transform(deduplicate)
)
# In ADF
# - Copy/paste transformation logic across pipelines
# - Or maintain custom activities (complex)
# - Or use stored procedures (not portable)
# - Technical debt accumulates