Microsoft Fabric

# Microsoft Fabric Lakehouse — PySpark notebook inside Fabric
# The notebook runs on Fabric Spark Runtime (same as Databricks)
# No cluster configuration needed — Fabric manages compute automatically

import pyspark.sql.functions as F
from pyspark.sql.types import *
from datetime import datetime

# In Fabric, you reference your Lakehouse files directly
# The Files/ section = raw storage (like ADLS Gen2 bronze layer)
# The Tables/ section = managed Delta Lake tables

# Read raw CSV from Lakehouse Files section (Bronze)
df_raw = spark.read     .option("header", True)     .option("inferSchema", True)     .csv("Files/bronze/sales/2025/03/15/sales_raw.csv")

print(f"Raw rows: {df_raw.count()}")

# Clean and transform (Silver logic)
df_silver = df_raw     .filter(F.col("order_id").isNotNull())     .dropDuplicates(["order_id"])     .withColumn("revenue",    F.col("revenue").cast(DoubleType()))     .withColumn("order_date", F.to_date(F.col("order_date"), "yyyy-MM-dd"))     .withColumn("load_ts",    F.lit(datetime.utcnow().isoformat()))

# Write to Lakehouse Tables section as Delta (Silver)
# This creates a table immediately queryable in SQL analytics endpoint
df_silver.write     .format("delta")     .mode("overwrite")     .saveAsTable("silver_sales")   # appears in Fabric SQL endpoint automatically

print("Written to silver_sales Delta table")

-- Microsoft Fabric Warehouse — T-SQL analytics
-- Fabric Warehouse is a full SQL engine on top of OneLake
-- Query Delta tables from Lakehouse directly — no data movement

-- Create Gold aggregation table in the Warehouse
CREATE TABLE gold_daily_revenue AS
SELECT
    order_date,
    region,
    product_category,
    COUNT(DISTINCT order_id)   AS total_orders,
    SUM(revenue)               AS total_revenue,
    AVG(revenue)               AS avg_order_value,
    COUNT(DISTINCT customer_id) AS unique_customers
FROM silver_sales          -- references Lakehouse Delta table via OneLake shortcut
GROUP BY order_date, region, product_category;

-- Cross-database query — Warehouse queries Lakehouse table directly
-- No COPY INTO, no ETL, no data movement
-- Both sit on the same OneLake storage

-- Create a view for Power BI (semantic layer)
CREATE VIEW vw_revenue_summary AS
SELECT
    order_date,
    region,
    SUM(total_revenue)  AS revenue,
    SUM(total_orders)   AS orders
FROM gold_daily_revenue
WHERE order_date >= DATEADD(month, -3, GETDATE())
GROUP BY order_date, region;

// Microsoft Fabric Data Pipeline — same as ADF, built into Fabric
// No separate ADF resource needed — pipelines live inside Fabric workspace

// Example: Copy Activity to ingest CSV from HTTP source into Lakehouse
{
  "name": "PL_Ingest_Sales_Daily",
  "activities": [
    {
      "name": "Copy_Sales_CSV",
      "type": "Copy",
      "source": {
        "type": "HttpSource",
        "requestMethod": "GET"
      },
      "sink": {
        "type": "LakehouseTableSink",
        "tableActionOption": "Append",
        "lakehouseTableName": "bronze_sales_raw"
      }
    },
    {
      "name": "Run_Notebook_Transform",
      "type": "TridentNotebook",
      "dependsOn": [{ "activity": "Copy_Sales_CSV", "dependencyConditions": ["Succeeded"] }],
      "typeProperties": {
        "notebookId": "your-notebook-id",
        "workspaceId": "your-workspace-id"
      }
    }
  ]
}