references/data_pipeline_architecture.md · 24000
etry_count >= MAX_RETRIES: dlq.send_to_dlq(message.value, e, {"retry_count": retry_count}) consumer.commit() else: raise # Will be retried
### Circuit Breaker
```python
from dataclasses import dataclass
from datetime import datetime, timedelta
from enum import Enum
import threading
class CircuitState(Enum):
CLOSED = "closed" # Normal operation
OPEN = "open" # Failing, reject calls
HALF_OPEN = "half_open" # Testing if recovered
@dataclass
class CircuitBreaker:
"""Circuit breaker for external service calls."""
failure_threshold: int = 5
recovery_timeout: timedelta = timedelta(seconds=30)
success_threshold: int = 3
def __post_init__(self):
self.state = CircuitState.CLOSED
self.failure_count = 0
self.success_count = 0
self.last_failure_time = None
self.lock = threading.Lock()
def call(self, func, *args, **kwargs):
"""Execute function with circuit breaker protection."""
with self.lock:
if self.state == CircuitState.OPEN:
if self._should_attempt_reset():
self.state = CircuitState.HALF_OPEN
else:
raise CircuitOpenError("Circuit is open")
try:
result = func(*args, **kwargs)
self._record_success()
return result
except Exception as e:
self._record_failure()
raise
def _record_success(self):
with self.lock:
if self.state == CircuitState.HALF_OPEN:
self.success_count += 1
if self.success_count >= self.success_threshold:
self.state = CircuitState.CLOSED
self.failure_count = 0
self.success_count = 0
elif self.state == CircuitState.CLOSED:
self.failure_count = 0
def _record_failure(self):
with self.lock:
self.failure_count += 1
self.last_failure_time = datetime.now()
if self.state == CircuitState.HALF_OPEN:
self.state = CircuitState.OPEN
self.success_count = 0
elif self.failure_count >= self.failure_threshold:
self.state = CircuitState.OPEN
def _should_attempt_reset(self):
if self.last_failure_time is None:
return True
return datetime.now() - self.last_failure_time >= self.recovery_timeout
# Usage
circuit = CircuitBreaker(failure_threshold=5, recovery_timeout=timedelta(seconds=60))
def call_external_api(data):
return circuit.call(external_api.process, data)
Data Ingestion Patterns
Change Data Capture (CDC)
# Using Debezium with Kafka Connect
# connector-config.json
{
"name": "postgres-cdc-connector",
"config": {
"connector.class": "io.debezium.connector.postgresql.PostgresConnector",
"database.hostname": "postgres",
"database.port": "5432",
"database.user": "debezium",
"database.password": "password",
"database.dbname": "source_db",
"database.server.name": "source",
"table.include.list": "public.orders,public.customers",
"plugin.name": "pgoutput",
"publication.name": "dbz_publication",
"slot.name": "debezium_slot",
"transforms": "unwrap",
"transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState",
"transforms.unwrap.drop.tombstones": "false"
}
}
Processing CDC Events:
def process_cdc_event(event):
"""Process Debezium CDC event."""
operation = event.get("op")
if operation == "c": # Create (INSERT)
after = event.get("after")
return {"action": "insert", "data": after}
elif operation == "u": # Update
before = event.get("before")
after = event.get("after")
return {"action": "update", "before": before, "after": after}
elif operation == "d": # Delete
before = event.get("before")
return {"action": "delete", "data": before}
elif operation == "r": # Read (snapshot)
after = event.get("after")
return {"action": "snapshot", "data": after}
Bulk Ingestion
# Efficient bulk loading to data warehouse
from concurrent.futures import ThreadPoolExecutor
import boto3
class BulkIngester:
"""Bulk ingest data to Snowflake via S3."""
def __init__(self, s3_bucket: str, snowflake_conn):
self.s3 = boto3.client('s3')
self.bucket = s3_bucket
self.snowflake = snowflake_conn
def ingest_dataframe(self, df, table_name: str, mode: str = "append"):
"""Bulk ingest DataFrame to Snowflake."""
# 1. Write to S3 as Parquet (compressed, columnar)
s3_path = f"s3://{self.bucket}/staging/{table_name}/{uuid.uuid4()}"
df.write.parquet(s3_path)
# 2. Create external stage if not exists
self.snowflake.execute(f"""
CREATE STAGE IF NOT EXISTS {table_name}_stage
URL = '{s3_path}'
CREDENTIALS = (AWS_KEY_ID='...' AWS_SECRET_KEY='...')
FILE_FORMAT = (TYPE = 'PARQUET')
""")
# 3. COPY INTO (much faster than INSERT)
if mode == "overwrite":
self.snowflake.execute(f"TRUNCATE TABLE {table_name}")
self.snowflake.execute(f"""
COPY INTO {table_name}
FROM @{table_name}_stage
FILE_FORMAT = (TYPE = 'PARQUET')
MATCH_BY_COLUMN_NAME = CASE_INSENSITIVE
ON_ERROR = 'CONTINUE'
""")
# 4. Cleanup staging files
self._cleanup_s3(s3_path)
Orchestration
Dependency Management
from airflow import DAG
from airflow.operators.python import PythonOperator
from airflow.sensors.external_task import ExternalTaskSensor
from airflow.utils.task_group import TaskGroup
from datetime import timedelta
with DAG("complex_pipeline") as dag:
# Wait for upstream DAG
wait_for_source = ExternalTaskSensor(
task_id="wait_for_source_etl",
external_dag_id="source_etl_dag",
external_task_id="final_task",
execution_delta=timedelta(hours=0),
timeout=3600,
mode="poke",
poke_interval=60,
)
# Parallel extraction group
with TaskGroup("extract") as extract_group:
extract_orders = PythonOperator(
task_id="extract_orders",
python_callable=extract_orders_func,
)
extract_customers = PythonOperator(
task_id="extract_customers",
python_callable=extract_customers_func,
)
extract_products = PythonOperator(
task_id="extract_products",
python_callable=extract_products_func,
)
# Sequential transformation
with TaskGroup("transform") as transform_group:
join_data = PythonOperator(
task_id="join_data",
python_callable=join_func,
)
aggregate = PythonOperator(
task_id="aggregate",
python_callable=aggregate_func,
)
join_data >> aggregate
# Load
load = PythonOperator(
task_id="load",
python_callable=load_func,
)
# Define dependencies
wait_for_source >> extract_group >> transform_group >> load
Dynamic DAG Generation
from airflow import DAG
from airflow.operators.python import PythonOperator
from datetime import datetime
import yaml
def create_etl_dag(config: dict) -> DAG:
"""Factory function to create ETL DAGs from config."""
dag = DAG(
dag_id=f"etl_{config['source']}_{config['destination']}",
schedule_interval=config.get('schedule', '@daily'),
start_date=datetime(2024, 1, 1),
catchup=False,
tags=['etl', 'auto-generated'],
)
with dag:
extract = PythonOperator(
task_id='extract',
python_callable=create_extract_func(config['source']),
)
transform = PythonOperator(
task_id='transform',
python_callable=create_transform_func(config['transformations']),
)
load = PythonOperator(
task_id='load',
python_callable=create_load_func(config['destination']),
)
extract >> transform >> load
return dag
# Load configurations
with open('/config/etl_pipelines.yaml') as f:
configs = yaml.safe_load(f)
# Generate DAGs
for config in configs['pipelines']:
dag_id = f"etl_{config['source']}_{config['destination']}"
globals()[dag_id] = create_etl_dag(config)