From 7494e1a51c2ac1a6285a933ed1fa52fb6a14769b Mon Sep 17 00:00:00 2001 From: Amit Kesarwani <93291915+kesarwam@users.noreply.github.com> Date: Thu, 18 Jun 2026 09:15:00 -0700 Subject: [PATCH 1/5] Added NetApp ONTAP sample --- .../netapp-ontap/.env.example | 37 ++ .../netapp-ontap/.gitignore | 30 ++ .../netapp-ontap/QUICK_START.md | 334 ++++++++++++++ 01_standalone_examples/netapp-ontap/README.md | 79 ++++ .../netapp-ontap/SETUP_GUIDE.md | 394 +++++++++++++++++ .../netapp-ontap/demo/DEMO_SCRIPT.md | 194 +++++++++ .../netapp-ontap/demo/demo_flow.py | 391 +++++++++++++++++ .../netapp-ontap/demo/requirements.txt | 1 + .../demo/seed_data/customers_baseline.csv | 21 + .../demo/seed_data/customers_experiment.csv | 21 + .../netapp-ontap/docker-compose.yml | 25 ++ .../netapp-ontap/lakefs.yaml.example | 47 ++ .../netapp-ontap/ontap-setup-notes.md | 408 ++++++++++++++++++ .../netapp-ontap/scripts/check-prereqs.sh | 141 ++++++ .../netapp-ontap/scripts/install-lakefs.sh | 28 ++ .../netapp-ontap/scripts/setup-demo.sh | 79 ++++ .../netapp-ontap/scripts/start-services.sh | 107 +++++ .../netapp-ontap/scripts/stop-services.sh | 30 ++ .../scripts/verify-connectivity.sh | 117 +++++ .../terraform/.terraform.lock.hcl | 44 ++ .../netapp-ontap/terraform/main.tf | 198 +++++++++ .../netapp-ontap/terraform/outputs.tf | 29 ++ .../terraform/terraform.tfvars.example | 15 + .../netapp-ontap/terraform/variables.tf | 29 ++ 24 files changed, 2799 insertions(+) create mode 100644 01_standalone_examples/netapp-ontap/.env.example create mode 100644 01_standalone_examples/netapp-ontap/.gitignore create mode 100644 01_standalone_examples/netapp-ontap/QUICK_START.md create mode 100644 01_standalone_examples/netapp-ontap/README.md create mode 100644 01_standalone_examples/netapp-ontap/SETUP_GUIDE.md create mode 100644 01_standalone_examples/netapp-ontap/demo/DEMO_SCRIPT.md create mode 100644 01_standalone_examples/netapp-ontap/demo/demo_flow.py create mode 100644 01_standalone_examples/netapp-ontap/demo/requirements.txt create mode 100644 01_standalone_examples/netapp-ontap/demo/seed_data/customers_baseline.csv create mode 100644 01_standalone_examples/netapp-ontap/demo/seed_data/customers_experiment.csv create mode 100644 01_standalone_examples/netapp-ontap/docker-compose.yml create mode 100644 01_standalone_examples/netapp-ontap/lakefs.yaml.example create mode 100644 01_standalone_examples/netapp-ontap/ontap-setup-notes.md create mode 100755 01_standalone_examples/netapp-ontap/scripts/check-prereqs.sh create mode 100755 01_standalone_examples/netapp-ontap/scripts/install-lakefs.sh create mode 100644 01_standalone_examples/netapp-ontap/scripts/setup-demo.sh create mode 100755 01_standalone_examples/netapp-ontap/scripts/start-services.sh create mode 100755 01_standalone_examples/netapp-ontap/scripts/stop-services.sh create mode 100755 01_standalone_examples/netapp-ontap/scripts/verify-connectivity.sh create mode 100644 01_standalone_examples/netapp-ontap/terraform/.terraform.lock.hcl create mode 100644 01_standalone_examples/netapp-ontap/terraform/main.tf create mode 100644 01_standalone_examples/netapp-ontap/terraform/outputs.tf create mode 100644 01_standalone_examples/netapp-ontap/terraform/terraform.tfvars.example create mode 100644 01_standalone_examples/netapp-ontap/terraform/variables.tf diff --git a/01_standalone_examples/netapp-ontap/.env.example b/01_standalone_examples/netapp-ontap/.env.example new file mode 100644 index 000000000..1610eddf4 --- /dev/null +++ b/01_standalone_examples/netapp-ontap/.env.example @@ -0,0 +1,37 @@ +# ───────────────────────────────────────────────────────────────────────────── +# Copy this file to .env and fill in all REPLACE_ME values. +# The .env file is gitignored and never committed. +# ───────────────────────────────────────────────────────────────────────────── + +# ── ONTAP S3 ───────────────────────────────────────────────────────────────── +# The IP of the ONTAP SVM data LIF you configured for S3 (port 80, HTTP). +# Example: http://192.168.233.20 +ONTAP_S3_ENDPOINT=http://REPLACE_ME + +# The bucket name you created on ONTAP (must match what you ran on ONTAP CLI). +ONTAP_S3_BUCKET=lakefs-data + +# Credentials output by: vserver object-store-server user create ... -user lakefs +ONTAP_S3_ACCESS_KEY=REPLACE_ME +ONTAP_S3_SECRET_KEY=REPLACE_ME + +# ── PostgreSQL (Docker) ─────────────────────────────────────────────────────── +POSTGRES_USER=lakefs +POSTGRES_PASSWORD=lakefs +POSTGRES_DB=lakefs + +# ── lakeFS Admin Credentials ───────────────────────────────────────────────── +# These become the lakeFS admin access key / secret key. +# You can use any alphanumeric strings. Keep them the same across restarts. +LAKEFS_INSTALLATION_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE +LAKEFS_INSTALLATION_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + +# Must be at least 32 characters. Generate one with: +# openssl rand -hex 20 +LAKEFS_AUTH_ENCRYPT_SECRET_KEY=REPLACE_WITH_32_CHAR_RANDOM_STRING + +# ── Demo Script Settings ────────────────────────────────────────────────────── +# These are read by demo/demo_flow.py +LAKEFS_HOST=http://localhost:8000 +LAKEFS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE +LAKEFS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY diff --git a/01_standalone_examples/netapp-ontap/.gitignore b/01_standalone_examples/netapp-ontap/.gitignore new file mode 100644 index 000000000..c141f9740 --- /dev/null +++ b/01_standalone_examples/netapp-ontap/.gitignore @@ -0,0 +1,30 @@ +# Secrets — never commit these +.env +lakefs.yaml +*.pid + +# lakeFS binary (downloaded locally) +bin/ + +# Terraform +.terraform/ +*.tfstate +*.tfstate.* +*.tfvars +!*.tfvars.example +crash.log +*.zip + +# Python +__pycache__/ +*.pyc +*.pyo +.venv/ +venv/ + +# macOS +.DS_Store + +# Editor +.vscode/ +.idea/ diff --git a/01_standalone_examples/netapp-ontap/QUICK_START.md b/01_standalone_examples/netapp-ontap/QUICK_START.md new file mode 100644 index 000000000..bcf4020b3 --- /dev/null +++ b/01_standalone_examples/netapp-ontap/QUICK_START.md @@ -0,0 +1,334 @@ +# ONTAP + lakeFS Demo — Quick Start + +**Goal:** Run a live demo of lakeFS managing data on NetApp ONTAP S3. + +**Use the AWS FSx for NetApp ONTAP path.** It is the supported path for this +PoC: provision with Terraform, then follow [SETUP_GUIDE.md](SETUP_GUIDE.md). + +```bash +cd terraform +cp terraform.tfvars.example terraform.tfvars # fill in your vpc/subnet/key/password +terraform init && terraform apply # ~20–30 min +# then follow SETUP_GUIDE.md Steps 4–6 +``` + +Cost: ~$8–10/day while running — **tear down after the demo**. +See **SETUP PATH 2** below for the summary and [SETUP_GUIDE.md](SETUP_GUIDE.md) +for the full runbook. + +--- + +# SETUP PATH 1: Local UTM Demo — ⚠️ DEPRECATED (reference only) + +> **Do not use this path.** Running the ONTAP simulator locally in UTM/VMware +> Fusion was abandoned — x86 emulation on Apple Silicon is too slow to be usable +> for a demo. It is kept here for reference only. Use the AWS FSx path above. + +## Phase 1 — Prerequisites (15 min) + +```bash +# Install tools +brew install --cask utm +brew install qemu +pip3 install requests + +# Verify +/Applications/UTM.app/Contents/MacOS/utmctl --version +qemu-img --version | head -1 +./scripts/check-prereqs.sh +``` + +## Phase 2 — Install lakeFS Binary (5 min) + +```bash +cd ~/code/Ontap_lakeFS_Integration +bash scripts/install-lakefs.sh +./bin/lakefs --version # expect: 1.80.0 +``` + +## Phase 3 — Import ONTAP VM (30 min) + +**Get the OVA file:** +- File: `~/Downloads/vsim-netapp-DOT9.18.1-cm_nodar.ova` (779 MB) +- If you don't have it, contact NetApp or use your existing copy + +**Extract and convert:** +```bash +mkdir -p /tmp/ontap-ova +cd /tmp/ontap-ova +tar -xf ~/Downloads/vsim-netapp-DOT9.18.1-cm_nodar.ova + +# Convert all 4 VMDKs to QCOW2 (takes ~3 minutes) +for vmdk in *.vmdk; do + qemu-img convert -f vmdk -O qcow2 "$vmdk" "${vmdk%.vmdk}.qcow2" +done +ls -lh *.qcow2 # should see 4 files +``` + +**Create the VM in UTM:** +1. Open UTM → click **+** +2. Select **Emulate** (not Virtualize) +3. Operating System: **Other** +4. Architecture: **x86_64** +5. System: **Standard PC (Q35 + ICH9)** +6. RAM: **6144 MB** | CPU Cores: **2** +7. Storage: **uncheck** "Create a new image drive" +8. Name: `ONTAP-sim-9.18.1` → **Save** (don't start) +9. Edit VM → **Drives** → delete any auto-created drive +10. Click **New Drive → Import** → select each `.qcow2` in order (disk1–4) + - Interface: **IDE** for each +11. Edit VM → **Network** → + - Mode: **Host Only** + - Card: **Intel Gigabit Ethernet (e1000)** + - Save + +**Boot the VM (10–15 min first boot):** + +Start the VM. Wait for the cluster wizard. Enter: + +``` +Cluster name: ontap-sim +Node name: (press Enter for default) + +Node management IP: 192.168.64.10 +Cluster management IP: 192.168.64.11 +Netmask: 255.255.255.0 +Gateway: 192.168.64.1 +DNS domain: lab.local +DNS servers: 8.8.8.8 +Admin password: NetApp1! +``` + +Wait for `ontap-sim::>` prompt. + +**Verify from Mac:** +```bash +ping -c 3 192.168.64.11 +ssh admin@192.168.64.11 # password: NetApp1! +``` + +## Phase 4 — Configure ONTAP S3 (10 min, inside ONTAP) + +SSH into ONTAP: +```bash +ssh admin@192.168.64.11 +``` + +Run these commands inside the ONTAP CLI: + +``` +# Check cluster +node show +storage aggregate show + +# Create S3 SVM +vserver create -vserver s3svm -subtype default +vserver modify -vserver s3svm -allowed-protocols s3 + +# Create S3 data LIF +network interface create \ + -vserver s3svm \ + -lif s3_data_1 \ + -service-policy data-s3-server \ + -home-node ontap-sim-01 \ + -home-port e0c \ + -address 192.168.64.20 \ + -netmask 255.255.255.0 \ + -status-admin up + +# Enable S3 server (HTTP only) +vserver object-store-server create \ + -vserver s3svm \ + -object-store-server s3.ontap.local \ + -is-http-enabled true \ + -is-https-enabled false \ + -port 80 + +# Create S3 user — COPY THE KEYS NOW +vserver object-store-server user create -vserver s3svm -user lakefs + +# Create bucket +vserver object-store-server bucket create \ + -vserver s3svm \ + -bucket lakefs-data \ + -size 50GB + +# Grant user access to bucket +vserver object-store-server bucket policy statement create \ + -vserver s3svm \ + -bucket lakefs-data \ + -effect allow \ + -action * \ + -principal lakefs \ + -resource lakefs-data,lakefs-data/* + +exit +``` + +**Test S3 from Mac:** +```bash +curl -s http://192.168.64.20/lakefs-data/ +# Should return XML or empty (not "connection refused") +``` + +## Phase 5 — Configure lakeFS (5 min) + +```bash +cd ~/code/Ontap_lakeFS_Integration + +# Copy config templates +cp .env.example .env +cp lakefs.yaml.example lakefs.yaml + +# Generate secret key +openssl rand -hex 20 +``` + +**Edit `.env`** — replace these values: +```bash +ONTAP_S3_ENDPOINT=http://192.168.64.20 +ONTAP_S3_BUCKET=lakefs-data +ONTAP_S3_ACCESS_KEY= +ONTAP_S3_SECRET_KEY= + +LAKEFS_AUTH_ENCRYPT_SECRET_KEY= +LAKEFS_HOST=http://localhost:8000 +``` + +**Edit `lakefs.yaml`** — replace these values: +```yaml +auth: + encrypt: + secret_key: "" + +blockstore: + s3: + endpoint: "http://192.168.64.20" + credentials: + access_key_id: "" + secret_access_key: "" +``` + +## Phase 6 — Start Services (5 min) + +```bash +bash scripts/start-services.sh +``` + +This starts PostgreSQL + lakeFS. Verify: +```bash +bash scripts/verify-connectivity.sh +# All items should show [OK] +``` + +Check lakeFS is running: +```bash +curl -s http://localhost:8000/api/v1/healthcheck +``` + +## Phase 7 — Run the Demo (3 min) + +```bash +source .env +python3 demo/demo_flow.py +``` + +You should see 11 steps printed. Open the UI: +``` +http://localhost:8000 +``` + +Login: Use the credentials you set in `.env` (default: `AKIAIOSFODNN7EXAMPLE`) + +--- + +# SETUP PATH 2: AWS FSx Demo (For customer presentations) + +See [SETUP_GUIDE.md](SETUP_GUIDE.md) for complete AWS setup instructions. + +**Summary:** +1. Create FSx ONTAP filesystem (20–30 min) +2. Launch EC2 instance +3. Configure ONTAP S3 (10 min via SSH) +4. Install lakeFS on EC2 (5 min) +5. Run demo script (3 min) + +Cost: ~$8–10/day. Tear down after demo. + +--- + +# Common Tasks + +## Run demo again +```bash +source .env && python3 demo/demo_flow.py +``` + +## Stop everything +```bash +bash scripts/stop-services.sh +``` +(Then suspend the UTM VM) + +## See lakeFS logs +```bash +tail -50 lakefs.log +``` + +## See PostgreSQL logs +```bash +docker logs postgres +``` + +## Restart services (if something breaks) +```bash +bash scripts/stop-services.sh +bash scripts/start-services.sh +``` + +## Check connectivity to ONTAP S3 +```bash +source .env +curl -s http://$ONTAP_S3_ENDPOINT/$ONTAP_S3_BUCKET/ +``` + +--- + +# Troubleshooting + +| Issue | Fix | +|-------|-----| +| lakeFS won't start | Check `lakefs.log`. Verify `.env` and `lakefs.yaml` don't have `REPLACE_ME` values | +| Can't reach ONTAP S3 | Ping `192.168.64.20`. Is UTM VM running? Check lakeFS logs | +| HTTP 403 from S3 | Recreate ONTAP user: `ssh admin@192.168.64.11` → `vserver object-store-server user delete -vserver s3svm -user lakefs` → `vserver object-store-server user create -vserver s3svm -user lakefs` (copy new keys) | +| Port 8000 in use | `lsof -i :8000` to find process, or edit `lakefs.yaml` to use a different port | +| PostgreSQL won't start | `docker logs postgres` to see error. Check `/tmp` disk space | +| ONTAP VM very slow | Normal for x86 emulation. First boot takes 10–15 min. Be patient | + +--- + +# What you end up with + +``` +Mac host +├── UTM VM (http://192.168.64.11) → ONTAP 9.18.1 Simulator +│ S3 endpoint: http://192.168.64.20 +│ Bucket: lakefs-data +├── Docker (localhost:5432) → PostgreSQL +└── Binary (localhost:8000) → lakeFS +``` + +**Demo flow:** +1. Python script (`demo_flow.py`) talks to lakeFS API +2. lakeFS creates repo and branches +3. Data is stored on ONTAP S3 at `http://192.168.64.20` +4. You can visualize everything in the lakeFS UI + +--- + +# For detailed info + +- **Overview & architecture:** [README.md](README.md) +- **AWS setup (main runbook):** [SETUP_GUIDE.md](SETUP_GUIDE.md) +- **ONTAP CLI reference (deprecated simulator path):** [ontap-setup-notes.md](ontap-setup-notes.md) +- **Demo script source:** `demo/demo_flow.py` diff --git a/01_standalone_examples/netapp-ontap/README.md b/01_standalone_examples/netapp-ontap/README.md new file mode 100644 index 000000000..d93ccbb62 --- /dev/null +++ b/01_standalone_examples/netapp-ontap/README.md @@ -0,0 +1,79 @@ +# lakeFS + NetApp ONTAP — Proof of Concept + +Spin up **lakeFS data versioning backed by a native NetApp ONTAP S3 bucket** on +**AWS FSx for NetApp ONTAP**, then run a short demo end to end. + +**Use case:** A data team adds a new ML feature (`payment_history_score`) to a +customer-churn dataset on a lakeFS branch, validates it, then merges to `main` — +all version-controlled in lakeFS, with the data physically stored on ONTAP S3. + +--- + +## Architecture + +``` +AWS VPC +├── FSx for NetApp ONTAP → SVM "fsx" → native S3 object-store server (HTTP) +│ Bucket: lakefs-data +├── EC2 (Ubuntu 22.04) → lakeFS (port 8000) + PostgreSQL (Docker, 5432) +└── Elastic IP → stable address for the lakeFS UI + +Demo script (Python) → lakeFS API → ONTAP S3 blockstore +``` + +lakeFS uses ONTAP's **native S3** as its blockstore over **HTTP (port 80)**, with +`force_path_style: true` (ONTAP S3 does not support virtual-hosted-style URLs). + +--- + +## What's in here + +| Path | Purpose | +|------|---------| +| `terraform/` | Provisions the AWS infra: FSx ONTAP filesystem + SVM + volume, EC2, security groups, Elastic IP | +| `SETUP_GUIDE.md` | **The main runbook.** Full step-by-step: provision → enable ONTAP S3 → install lakeFS → run demo → tear down | +| `demo/` | The Python demo (`demo_flow.py`), seed data, and `DEMO_SCRIPT.md` talk track | +| `lakefs.yaml.example`, `.env.example` | Config templates (copy and fill in) | +| `scripts/` | Helper scripts (connectivity checks, service start/stop) | + +> ⚠️ **Note on what Terraform does *not* do.** Terraform provisions the raw +> infrastructure only. Enabling the ONTAP S3 server, creating the S3 user/bucket, +> and installing lakeFS are **manual post-provisioning steps** done over SSH — +> see `SETUP_GUIDE.md` Steps 4–6. (AWS does not expose ONTAP S3 enablement +> through the FSx API.) + +--- + +## Quickstart + +Prerequisites: an AWS account (default region `us-east-1`), an existing EC2 key +pair, a VPC with a public subnet, and the AWS CLI + Terraform installed. + +```bash +cd terraform +cp terraform.tfvars.example terraform.tfvars +# Edit terraform.tfvars: set vpc_id, subnet_id, key_pair_name, fsxadmin_password + +terraform init +terraform apply # ~20–30 min, mostly waiting for FSx to provision +``` + +Terraform outputs the EC2 IP, lakeFS UI URL, and the FSx/SVM management IPs. +Then follow **`SETUP_GUIDE.md` from Step 4** to: + +1. Enable the ONTAP S3 server and create the `lakefs` S3 user + `lakefs-data` bucket (SSH into ONTAP). +2. Install and configure lakeFS on the EC2 host (pointing its blockstore at the SVM management IP). +3. Open the lakeFS UI and run `demo/demo_flow.py`. + +When you're done, **tear everything down** (`SETUP_GUIDE.md` Part 3) — the +environment costs ~$9/day while running. + +--- + +## Deprecated: local ONTAP simulator path + +An earlier approach ran the **ONTAP 9.18.1 simulator locally** in UTM/VMware +Fusion on a Mac (see `ontap-setup-notes.md`). **This was abandoned** — x86 +emulation of the simulator on Apple Silicon is far too slow to be usable for a +demo. `ontap-setup-notes.md` is kept for reference only; **use the AWS FSx path +above.** diff --git a/01_standalone_examples/netapp-ontap/SETUP_GUIDE.md b/01_standalone_examples/netapp-ontap/SETUP_GUIDE.md new file mode 100644 index 000000000..068f899fb --- /dev/null +++ b/01_standalone_examples/netapp-ontap/SETUP_GUIDE.md @@ -0,0 +1,394 @@ +# lakeFS + NetApp ONTAP Demo — Setup Guide + +## Overview + +This guide sets up a live demo of lakeFS running against a native NetApp ONTAP S3 bucket on AWS FSx. + +**Cost:** ~$8/day while running. **Tear it down after the demo** — total cost per demo ~$10. + +**Time to set up:** ~35 minutes (mostly waiting for FSx to provision) + +--- + +## Prerequisites + +- AWS account access (us-east-1) +- The `lakefs-ontap-demo.pem` key file +- A terminal (Mac: Terminal app) + +--- + +## PART 1 — Build the Infrastructure (30 min, mostly waiting) + +### Step 1 — Create FSx for NetApp ONTAP + +1. Go to **AWS Console → FSx → Create file system** +2. Select **Amazon FSx for NetApp ONTAP** → Next +3. Select **Standard create** +4. Fill in: + - File system name: `lakefs-ontap-demo` + - Deployment type: **Single-AZ 2** + - SSD storage: `1024` GiB + - Throughput: **Recommended (384 MB/s)** + - VPC: `demo-lakefs-vpc` + - Subnet: `demo-lakefs-ontap-public` (`subnet-05960cb8e849935c0`) + - VPC Security Groups: leave default + - File system admin password: `Netapp1!` + - SVM name: `fsx` + - SVM admin password: `Netapp1!` + - Storage efficiency: **Enabled** + - Everything else: leave as default +5. Click **Create file system** +6. ⏳ Wait 20-30 minutes for status to show **Available** + +> While waiting, continue with Steps 2 and 3. + +--- + +### Step 2 — Launch EC2 Instance + +1. Go to **AWS Console → EC2 → Launch Instance** +2. Fill in: + - Name: `lakefs-ontap-demo` + - AMI: **Ubuntu Server 22.04 LTS** + - Instance type: **t3.small** + - Key pair: `lakefs-ontap-demo` + - VPC: `demo-lakefs-vpc` + - Subnet: `demo-lakefs-ontap-public` + - Auto-assign public IP: **Enable** + - Security group: select existing `lakefs-ontap-demo-sg` + - (If it doesn't exist, create new with: SSH/22, TCP/8000, HTTPS/443 — all **My IP**) +3. Click **Launch instance** +4. Go to **EC2 → Elastic IPs → Allocate → Associate** to the new instance + (This keeps the IP stable across stops/starts) + +--- + +### Step 3 — Configure FSx Security Group and NLB + +**Security group** (allows EC2 → ONTAP traffic): + +1. Go to **EC2 → Security Groups** +2. Find the FSx security group (the one attached to your FSx filesystem) +3. **Edit inbound rules → Add rule:** + - Type: All traffic + - Source: `lakefs-ontap-demo-sg` +4. Save rules + +**Network Load Balancer** (exposes ONTAP S3 publicly for pre-signed URL support): + +5. **Edit inbound rules → Add another rule:** + - Type: TCP, Port: 80 + - Source: `0.0.0.0/0` + - Description: `ONTAP S3 HTTP via NLB for pre-signed URLs` +6. Go to **EC2 → Load Balancers → Create load balancer → Network Load Balancer** + - Name: `lakefs-ontap-s3` + - Scheme: **Internet-facing** + - Listener: TCP port 80 + - Target group: IP type, TCP port 80, target = SVM management IP (e.g. `10.20.10.188`) +7. Note the NLB DNS name — you'll need it for the lakeFS config (`pre_signed_endpoint`) + +> **Terraform users:** Steps 3–7 are handled automatically by `terraform apply`. + +--- + +### Step 4 — Configure ONTAP S3 + +Once FSx shows **Available**: + +1. Go to **FSx → lakefs-ontap-demo → Administration tab** +2. Note the **Management endpoint IP address** (e.g. `10.0.10.251`) + +3. SSH into EC2: +```bash +ssh -i ~/path/to/lakefs-ontap-demo.pem ubuntu@ +``` + +4. From EC2, SSH into ONTAP: +```bash +ssh fsxadmin@ +# Password: Netapp1! +``` + +5. Run these commands in order: + +``` +# Check if S3 server already exists +vserver object-store-server show -vserver fsx +``` + +If no S3 server exists, create one: +``` +vserver object-store-server create -vserver fsx -object-store-server s3.demo -is-http-enabled true -is-https-enabled false +``` + +Create S3 user: +``` +vserver object-store-server user create -vserver fsx -user lakefs +``` +⚠️ **IMPORTANT: Copy the Access Key and Secret Key printed here — you won't see them again!** + +Create bucket: +``` +vserver object-store-server bucket create -vserver fsx -bucket lakefs-data -size 500GB +``` + +Set bucket policy: +``` +vserver object-store-server bucket policy statement create -vserver fsx -bucket lakefs-data -effect allow -action * -principal lakefs -resource lakefs-data,lakefs-data/* +``` + +Type `exit` to leave ONTAP CLI. + +--- +### Step 5 — Install and Configure lakeFS OSS or lakeFS Enterprise on EC2 + + +### Step 5a — Install and Configure lakeFS OSS on EC2 + +Still on EC2, run: + +```bash +# Install Docker and Python +sudo apt-get update && sudo apt-get install -y docker.io python3-pip +sudo systemctl start docker +sudo usermod -aG docker ubuntu +newgrp docker + +# Start Postgres +docker run -d --name postgres \ + -e POSTGRES_USER=lakefs \ + -e POSTGRES_PASSWORD=lakefs \ + -e POSTGRES_DB=lakefs \ + -p 5432:5432 \ + --restart unless-stopped \ + postgres:15-alpine + +# Install lakeFS +LAKEFS_VERSION=$(curl -s https://api.github.com/repos/treeverse/lakeFS/releases/latest | grep '"tag_name"' | cut -d'"' -f4 | sed 's/v//') +curl -L "https://github.com/treeverse/lakeFS/releases/download/v${LAKEFS_VERSION}/lakeFS_${LAKEFS_VERSION}_Linux_x86_64.tar.gz" | tar xz +sudo mv lakefs /usr/local/bin/ +``` + +Get the SVM management IP from FSx console → Storage Virtual Machines → fsx → Endpoints → Management IP address + +Create the config file (replace the values in CAPS): +```bash +cat > ~/lakefs.yaml << 'EOF' +database: + type: postgres + postgres: + connection_string: "postgresql://lakefs:lakefs@localhost:5432/lakefs?sslmode=disable" + +auth: + encrypt: + secret_key: "a7f3d9e2b8c4f1a6e5d0b3c7f2a9e4d1b6c8f3a2" + +installation: + user_name: admin + access_key_id: "AKIAIOSFODNN7EXAMPLE" + secret_access_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + +blockstore: + type: s3 + s3: + endpoint: "http://REPLACE_WITH_SVM_MANAGEMENT_IP" + force_path_style: true + region: "us-east-1" + credentials: + access_key_id: "REPLACE_WITH_ONTAP_ACCESS_KEY" + secret_access_key: "REPLACE_WITH_ONTAP_SECRET_KEY" + +logging: + level: "INFO" + format: "text" +EOF +``` + +Start lakeFS: +```bash +nohup lakefs run --config ~/lakefs.yaml > ~/lakefs.log 2>&1 & +``` + +--- + +### Step 5b — Install and Configure lakeFS Enterprise on EC2 + +# Copy lakeFS Enterprise license file to EC2: +```bash +scp -i license.token ubuntu@:/home/ubuntu/license.token +``` + + +Still on EC2, run: + +```bash +# Install Docker and Python +sudo apt-get update && sudo apt-get install -y docker.io python3-pip +sudo systemctl start docker +sudo usermod -aG docker ubuntu +newgrp docker + +# Start Postgres +docker run -d --name postgres \ + -e POSTGRES_USER=lakefs \ + -e POSTGRES_PASSWORD=lakefs \ + -e POSTGRES_DB=lakefs \ + -p 5432:5432 \ + --restart unless-stopped \ + postgres:15-alpine + +# Install lakeFS Enterprise +LAKEFS_VERSION=REPLACE_WITH_LAKEFS_ENTERPRISE_VERSION_NO +curl -L "https://artifacts.lakefs.io/lakefs-enterprise/${LAKEFS_VERSION}/lakefs-enterprise_${LAKEFS_VERSION}_Linux_x86_64.tar.gz" | +tar xz +sudo mv lakefs /usr/local/bin/ +``` + +Get the SVM management IP from FSx console → Storage Virtual Machines → fsx → Endpoints → Management IP address. +Get the NLB DNS from `terraform output ontap_s3_endpoint` or the AWS console. + +Create the config file (replace the values in CAPS): +```bash +cat > ~/lakefs.yaml << 'EOF' +database: + type: postgres + postgres: + connection_string: "postgresql://lakefs:lakefs@localhost:5432/lakefs?sslmode=disable" + +auth: + encrypt: + secret_key: "a7f3d9e2b8c4f1a6e5d0b3c7f2a9e4d1b6c8f3a2" + +installation: + user_name: admin + access_key_id: "AKIAIOSFODNN7EXAMPLE" + secret_access_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + +blockstore: + type: s3 + s3: + endpoint: "http://REPLACE_WITH_SVM_MANAGEMENT_IP" + pre_signed_endpoint: "http://REPLACE_WITH_NLB_DNS" + force_path_style: true + region: "us-east-1" + credentials: + access_key_id: "REPLACE_WITH_ONTAP_ACCESS_KEY" + secret_access_key: "REPLACE_WITH_ONTAP_SECRET_KEY" + +logging: + level: "INFO" + format: "text" + +license: + path: /home/ubuntu/license.token +features: + local_rbac: true +auth: + ui_config: + rbac: internal +EOF +``` + +Start lakeFS: +```bash +nohup lakefs run --config ~/lakefs.yaml > ~/lakefs.log 2>&1 & +``` + +--- + +### Step 6 — Complete lakeFS Setup + +1. Open browser: `http://:8000/setup` +2. Click through setup — it will generate an **Access Key ID** and **Secret Key** +3. **Save these credentials** — you'll need them to run the demo + +--- + +## PART 2 — Run the Demo + +### Before Every Demo Run + +SSH into EC2 and make sure lakeFS is running: +```bash +ssh -i ~/path/to/lakefs-ontap-demo.pem ubuntu@ +curl -s http://localhost:8000/api/v1/healthcheck +``` + +If no response, restart lakeFS: +```bash +nohup lakefs run --config ~/lakefs.yaml > ~/lakefs.log 2>&1 & +``` + +### Run the demo script: +```bash +cd ~/demo/demo +LAKEFS_ACCESS_KEY_ID= \ +LAKEFS_SECRET_ACCESS_KEY= \ +python3 demo_flow.py +``` + +> If it fails with "storage namespace already in use", the script will auto-increment the version (v2, v3, etc.) + +### Open these browser tabs: +1. `http://:8000/repositories` — lakeFS UI +2. AWS Console → FSx → `lakefs-ontap-demo` — ONTAP storage view + +### To show raw S3 objects on ONTAP (optional, great for technical audiences): + +From EC2, install AWS CLI if not already installed: +```bash +pip3 install awscli --upgrade +``` + +Then list all objects lakeFS wrote to ONTAP S3: +```bash +AWS_ACCESS_KEY_ID= \ +AWS_SECRET_ACCESS_KEY="" \ +AWS_DEFAULT_REGION=us-east-1 \ +aws s3 ls s3://lakefs-data/ \ + --recursive \ + --endpoint-url http:// +``` + +This shows three levels of the stack: +- **lakeFS UI** → logical dataset with branches/commits +- **S3 objects** → content-addressed blocks on ONTAP +- **ONTAP CLI** (`volume show`) → physical FlexGroup volume (`fg_oss_*`) storing the data + +--- + +## PART 3 — Tear Down (After Demo) + +**Delete in this order:** + +1. **FSx Volume:** FSx → lakefs-ontap-demo → Volumes → vol1 → Delete +2. **FSx SVM:** FSx → lakefs-ontap-demo → Storage Virtual Machines → fsx → Delete +3. **FSx Filesystem:** FSx → lakefs-ontap-demo → Actions → Delete +4. **EC2:** EC2 → Instances → lakefs-ontap-demo → Terminate +5. **Elastic IP:** EC2 → Elastic IPs → Release (otherwise you're charged for unused EIP) + +--- + +## PART 4 — Costs Reference + +| Resource | Cost | +|----------|------| +| FSx ONTAP (1TB, 384MB/s) | ~$8.40/day | +| EC2 t3.small | ~$0.50/day | +| Elastic IP (when attached) | Free | +| **Total per demo day** | **~$9/day** | + +**Tip:** Spin up the morning of the demo, tear down the same evening = ~$10 total. + +--- + +## Troubleshooting + +| Problem | Fix | +|---------|-----| +| SSH times out | Your IP changed — update security group rules to **My IP** | +| lakeFS UI unreachable | SSH into EC2 and restart lakeFS (see above) | +| Demo script fails with "namespace in use" | Normal on re-run — script uses a new prefix automatically | +| Can't SSH to ONTAP | Must SSH from EC2 (private IP), not from your Mac | +| ONTAP password rejected | Password is `Netapp1!` for both fsxadmin and vsadmin | diff --git a/01_standalone_examples/netapp-ontap/demo/DEMO_SCRIPT.md b/01_standalone_examples/netapp-ontap/demo/DEMO_SCRIPT.md new file mode 100644 index 000000000..aca022054 --- /dev/null +++ b/01_standalone_examples/netapp-ontap/demo/DEMO_SCRIPT.md @@ -0,0 +1,194 @@ +# lakeFS + NetApp ONTAP — Phase 1 Live Demo Script + +**Use case:** ML Feature Engineering for Customer Churn Prediction +**Duration:** ~10 minutes +**Tabs to have open before you start:** +1. AWS Console → FSx → `lakefs-ontap-demo` +2. lakeFS UI → `http://52.87.202.16:8000` +3. Terminal (SSH into EC2, lakeFS running) + +--- + +## Opening (30 seconds) + +> "Today I'm going to show you how lakeFS brings Git-like data versioning +> to NetApp ONTAP storage — so your ML teams can branch, commit, diff, +> and merge datasets the same way engineers do with code. +> No data copies, no manual versioning, no 'which CSV is the right one' chaos." + +--- + +## Tab 1 — AWS FSx Console (2 min) + +**Show the filesystem summary page.** + +> "This is real NetApp ONTAP — running as a managed service on AWS. +> Same ONTAP you'd run on-prem, same CLI, same storage efficiency features. +> One terabyte of SSD, 384 MB/s throughput, storage efficiency enabled — +> deduplication, compression, compaction all on by default." + +**Click Storage Virtual Machines → fsx → Endpoints tab.** + +> "ONTAP exposes multiple protocols from a single SVM — +> NFS, iSCSI, and crucially for us: native S3. +> No translation layer, no gateway — ONTAP speaks S3 natively on port 80. +> This is what lakeFS connects to as its backing store." + +**Click Volumes tab.** + +> "You can see the vol1 FlexVol here. This is where the actual data lives. +> Everything lakeFS writes goes through the ONTAP S3 endpoint, +> gets stored here, and benefits from all of ONTAP's enterprise capabilities." + +--- + +## Tab 2 — lakeFS UI (5 min) + +**Navigate to `http://52.87.202.16:8000` → churn-features repository.** + +> "Now let's look at lakeFS. This is our repository — think of it like +> a GitHub repo, but instead of code files, it contains ML training data. +> The physical storage is that ONTAP S3 bucket we just looked at." + +**Click on the `main` branch → data/ → customers.csv** + +> "This is our baseline dataset — 20 customer records with features like +> tenure, monthly charges, contract type. This is what the model was +> originally trained on. It's committed, versioned, immutable." + +**Switch to the `feature-add-payment-history` branch.** + +> "Now here's where it gets interesting. A data scientist wanted to +> engineer a new feature — payment_history_score — derived from billing data. +> Instead of overwriting the original dataset, they created a branch. +> Zero data copied. lakeFS uses copy-on-write — the branch costs nothing +> until something actually changes." + +**Click on data/ → customers.csv on the branch.** + +> "On this branch, the dataset has the new payment_history_score column. +> The original main branch is completely untouched." + +**Click Commits tab on the branch.** + +> "Full commit history. Every change has a hash, a timestamp, a committer, +> and rich metadata. This one has experiment_id=exp-2025-001. +> You can always answer: what data was used to train this model, and when?" + +**Click Compare → select main as the base.** + +> "This is the diff. Exactly one file changed: data/customers.csv. +> One column added. This is how you do code review for data — +> before you merge a new feature into your production dataset, +> you can see exactly what changed." + +**Click on the main branch → Commits tab.** + +> "After the team validated the experiment — AUC improved — they merged +> the branch back to main. Just like a pull request. The merge commit +> is right here: 'payment_history_score feature — validated, AUC +0.03'. +> Main now has the engineered feature. And we have a complete audit trail +> of how we got here." + +--- + +## Tab 3 — ONTAP CLI (2 min, optional for technical audiences) + +**Switch to the terminal and SSH into EC2, then SSH to ONTAP:** +``` +ssh fsxadmin@10.0.10.251 +``` + +Password: `Netapp1!` + +> "For the technically curious — let's look at this from the ONTAP side." + +``` +vserver object-store-server bucket show -vserver fsx -instance +``` + +> "There's our lakefs-data bucket — native ONTAP S3, no gateway." + +``` +vserver object-store-server bucket usage show -vserver fsx +``` + +> "Bucket usage — every byte lakeFS wrote is accounted for here." + +``` +volume show -vserver fsx -fields size,used,available +``` + +> "Three volumes. The fg_oss_* FlexGroup is where ONTAP physically stores +> the S3 objects — that's where all the lakeFS data lives on disk. +> Notice vol1 is nearly empty — the S3 data goes into the object store volume." + +``` +volume efficiency show -vserver fsx +``` + +> "Storage efficiency running automatically — dedup and compression +> on everything lakeFS writes, with no configuration required." + +**Then exit ONTAP and show the raw S3 objects from EC2:** +``` +exit +``` + +```bash +AWS_ACCESS_KEY_ID=WD45C411GSC72OQ8RFPI \ +AWS_SECRET_ACCESS_KEY="HXtcC13CH_qBrq06SBY4Ng5gYGCib6Lq_AN9rN9_" \ +AWS_DEFAULT_REGION=us-east-1 \ +aws s3 ls s3://lakefs-data/ \ + --recursive \ + --endpoint-url http://10.0.10.232 +``` + +> "Three levels of visibility in one demo: +> The logical dataset in lakeFS, the S3 objects on ONTAP, +> and the physical blocks in the FlexGroup volume. +> Full transparency from dataset to disk." + +--- + +## Closing (30 seconds) + +> "So what did we just see? +> +> NetApp ONTAP provides enterprise-grade S3 storage — on-prem or in the cloud. +> lakeFS sits on top and adds the version control layer that data teams need: +> branches for safe experimentation, commits for reproducibility, +> diffs for data review, merges for controlled promotion to production. +> +> No infrastructure changes. No data copies. No workflow disruption. +> Your data engineers work in Python, Spark, or SQL — they don't need to +> know anything about lakeFS internals. It just looks like S3 to them. +> +> And your storage team keeps running ONTAP — with all the efficiency, +> protection, and compliance features they already rely on." + +--- + +## Q&A Prep + +**Q: Does lakeFS work with on-prem ONTAP, not just FSx?** +> Yes — any ONTAP with S3 enabled. The configuration is identical. +> On-prem, FSx, or Cloud Volumes ONTAP. + +**Q: What happens to storage when you create branches?** +> Copy-on-write — a branch costs zero storage until data changes. +> lakeFS uses content addressing, so identical blocks are never stored twice. + +**Q: Can we plug this into our existing ML pipelines?** +> Yes. lakeFS exposes an S3-compatible endpoint. Any tool that reads S3 +> works with lakeFS — Spark, pandas, SageMaker, Databricks, all of it. + +**Q: What about access control?** +> lakeFS has its own RBAC on top of ONTAP's native access controls. +> You can lock down who can merge to main, who can create branches, etc. + +**Q: How is this different from just using S3 versioning?** +> S3 versioning versions individual objects. lakeFS versions entire datasets +> atomically — a commit captures the state of thousands of files at once. +> You get cross-file consistency, branching, merging, and diffs. +> S3 versioning can't give you that. diff --git a/01_standalone_examples/netapp-ontap/demo/demo_flow.py b/01_standalone_examples/netapp-ontap/demo/demo_flow.py new file mode 100644 index 000000000..c0d99535d --- /dev/null +++ b/01_standalone_examples/netapp-ontap/demo/demo_flow.py @@ -0,0 +1,391 @@ +#!/usr/bin/env python3 +""" +Phase 1 Demo: lakeFS + NetApp ONTAP S3 +Use case: Customer Churn — ML Feature Engineering with Dataset Versioning + +Demonstrates: + 1. Repository creation backed by native ONTAP S3 + 2. Ingesting a baseline ML training dataset + 3. Creating an experiment branch + 4. Engineering a new feature on the branch + 5. Committing with rich metadata + 6. Diff between branch and main + 7. Inspecting commit history + 8. Merging the validated feature back to main + +Run from the demo/ directory (or project root): + python3 demo/demo_flow.py + +Prerequisites: + - lakeFS running at http://localhost:8000 + - .env loaded (or env vars set directly) + - See README.md for full setup steps. +""" + +import os +import sys +import json +import time +import textwrap +from pathlib import Path + +import requests +from requests.auth import HTTPBasicAuth + +# ───────────────────────────────────────────────────────────────────────────── +# Configuration — reads from environment (set via .env / source .env) +# ───────────────────────────────────────────────────────────────────────────── + +LAKEFS_HOST = os.getenv("LAKEFS_HOST", "http://localhost:8000") +LAKEFS_ACCESS_KEY = os.getenv("LAKEFS_ACCESS_KEY_ID", "AKIAIOSFODNN7EXAMPLE") +LAKEFS_SECRET_KEY = os.getenv("LAKEFS_SECRET_ACCESS_KEY","wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY") +ONTAP_S3_BUCKET = os.getenv("ONTAP_S3_BUCKET", "lakefs-data") + +REPO_NAME = "churn-features" +MAIN_BRANCH = "main" +EXP_BRANCH = "feature-add-payment-history" +STORAGE_NS = f"s3://{ONTAP_S3_BUCKET}/{REPO_NAME}" + +BASE_URL = f"{LAKEFS_HOST}/api/v1" +AUTH = HTTPBasicAuth(LAKEFS_ACCESS_KEY, LAKEFS_SECRET_KEY) +SEED_DIR = Path(__file__).parent / "seed_data" + + +# ───────────────────────────────────────────────────────────────────────────── +# Helpers +# ───────────────────────────────────────────────────────────────────────────── + +def banner(title: str): + width = 62 + print(f"\n╔{'═' * width}╗") + print(f"║ {title:<{width - 2}}║") + print(f"╚{'═' * width}╝") + + +def step(n: int, total: int, msg: str): + print(f"\n── Step {n}/{total}: {msg} {'─' * max(1, 50 - len(msg))}") + + +def ok(msg: str): + print(f" ✓ {msg}") + + +def info(msg: str): + print(f" → {msg}") + + +def fatal(msg: str): + print(f"\n ✗ ERROR: {msg}") + print(" See README.md — Troubleshooting section.\n") + sys.exit(1) + + +def api(method: str, path: str, **kwargs) -> requests.Response: + """Authenticated lakeFS REST call. Raises on non-2xx.""" + url = f"{BASE_URL}{path}" + resp = requests.request(method, url, auth=AUTH, timeout=30, **kwargs) + if not resp.ok: + print(f"\n ✗ API {method} {path} → HTTP {resp.status_code}") + try: + detail = resp.json() + print(f" {json.dumps(detail, indent=6)}") + except Exception: + print(f" {resp.text[:300]}") + resp.raise_for_status() + return resp + + +# ───────────────────────────────────────────────────────────────────────────── +# Demo steps +# ───────────────────────────────────────────────────────────────────────────── + +def check_lakefs(): + """Verify lakeFS is running and credentials work.""" + try: + api("GET", "/healthcheck") + except requests.exceptions.ConnectionError: + fatal( + f"Cannot connect to lakeFS at {LAKEFS_HOST}\n" + " Is lakeFS running? bash scripts/start-services.sh" + ) + except requests.exceptions.HTTPError as e: + if e.response.status_code == 401: + fatal("Authentication failed — check LAKEFS_ACCESS_KEY_ID / LAKEFS_SECRET_ACCESS_KEY") + raise + + resp = api("GET", "/config") + cfg = resp.json() + blockstore = cfg.get("storage_config", {}).get("blockstore_type", "unknown") + ok(f"lakeFS is reachable at {LAKEFS_HOST}") + ok(f"Blockstore type: {blockstore} (should be 's3' → ONTAP)") + + +def create_repository(): + """Create (or re-create) the demo repository backed by ONTAP S3.""" + info(f"Repository : {REPO_NAME}") + info(f"Default branch: {MAIN_BRANCH}") + + # Delete if it already exists so the demo is idempotent + try: + api("DELETE", f"/repositories/{REPO_NAME}") + info("Previous run detected — deleted existing repository (clean start)") + except requests.exceptions.HTTPError as e: + if e.response.status_code != 404: + raise + + # Try base namespace first, then increment version if namespace is already in use + base_ns = STORAGE_NS + storage_ns = base_ns + for attempt in range(1, 20): + try: + api("POST", "/repositories", json={ + "name": REPO_NAME, + "storage_namespace": storage_ns, + "default_branch": MAIN_BRANCH, + }) + break + except requests.exceptions.HTTPError as e: + if e.response.status_code == 400 and "already in use" in e.response.text: + storage_ns = f"{base_ns}-v{attempt + 1}" + info(f"Namespace in use — retrying with: {storage_ns}") + else: + raise + + ok(f"Repository '{REPO_NAME}' created") + ok(f"Data will be stored at: {storage_ns} (on ONTAP S3 bucket '{ONTAP_S3_BUCKET}')") + + +def upload_baseline(): + """Upload the baseline customer churn dataset to the main branch.""" + csv_file = SEED_DIR / "customers_baseline.csv" + size_kb = csv_file.stat().st_size / 1024 + info(f"File: {csv_file.name} ({size_kb:.1f} KB, 20 customer records)") + info(f"Target: lakefs://{REPO_NAME}/{MAIN_BRANCH}/data/customers.csv") + + with open(csv_file, "rb") as fh: + api( + "POST", + f"/repositories/{REPO_NAME}/branches/{MAIN_BRANCH}/objects", + params={"path": "data/customers.csv"}, + files={"content": ("customers.csv", fh, "text/csv")}, + ) + ok("Uploaded data/customers.csv to main branch") + + +def commit_to_main() -> str: + """Commit the baseline upload on main with descriptive metadata.""" + resp = api( + "POST", + f"/repositories/{REPO_NAME}/branches/{MAIN_BRANCH}/commits", + json={ + "message": "feat: add baseline customer churn features (20 records, v1.0)", + "metadata": { + "source": "crm_export", + "version": "1.0", + "rows": "20", + "columns": "tenure_months,monthly_charges,total_charges,contract_type," + "payment_method,tech_support,internet_service,num_products,churned", + "owner": "data-team", + }, + }, + ) + commit_id = resp.json()["id"] + ok(f"Committed on main → {commit_id[:16]}…") + info("Metadata: source=crm_export, version=1.0, rows=20") + return commit_id + + +def create_experiment_branch(): + """Branch off main for the payment-history feature experiment.""" + info(f"New branch : {EXP_BRANCH}") + info(f"From : {MAIN_BRANCH} (isolated copy, no data duplication)") + + api("POST", f"/repositories/{REPO_NAME}/branches", json={ + "name": EXP_BRANCH, + "source": MAIN_BRANCH, + }) + ok(f"Branch '{EXP_BRANCH}' created") + ok("lakeFS uses copy-on-write — zero extra storage until data changes") + + +def upload_experiment_data(): + """Upload the feature-engineered CSV to the experiment branch.""" + csv_file = SEED_DIR / "customers_experiment.csv" + size_kb = csv_file.stat().st_size / 1024 + info(f"File: {csv_file.name} ({size_kb:.1f} KB)") + info("Change: added column 'payment_history_score' (engineered from billing data)") + info(f"Target: lakefs://{REPO_NAME}/{EXP_BRANCH}/data/customers.csv") + + with open(csv_file, "rb") as fh: + api( + "POST", + f"/repositories/{REPO_NAME}/branches/{EXP_BRANCH}/objects", + params={"path": "data/customers.csv"}, + files={"content": ("customers.csv", fh, "text/csv")}, + ) + ok("Uploaded modified data/customers.csv to experiment branch") + + +def commit_on_branch() -> str: + """Commit the feature engineering experiment on the branch.""" + resp = api( + "POST", + f"/repositories/{REPO_NAME}/branches/{EXP_BRANCH}/commits", + json={ + "message": "feat: add payment_history_score feature (engineered from billing data)", + "metadata": { + "experiment_id": "exp-2025-001", + "feature_added": "payment_history_score", + "derivation": "rolling_30d_on_time_payment_ratio", + "auc_improvement": "0.03", + "owner": "data-team", + "status": "pending-review", + }, + }, + ) + commit_id = resp.json()["id"] + ok(f"Committed on branch → {commit_id[:16]}…") + info("Metadata: experiment_id=exp-2025-001, feature=payment_history_score") + return commit_id + + +def show_diff(): + """Show what changed between the experiment branch and main.""" + resp = api( + "GET", + f"/repositories/{REPO_NAME}/refs/{EXP_BRANCH}/diff/{MAIN_BRANCH}", + ) + results = resp.json().get("results", []) + + if results: + ok(f"{len(results)} change(s) detected:") + for item in results: + path = item.get("path", "?") + change = item.get("type", "?").upper() + size_bytes = item.get("size_bytes", None) + size_str = f" ({size_bytes:,} bytes)" if size_bytes else "" + print(f" [{change:8s}] {path}{size_str}") + else: + info("No differences (unexpected — check upload steps)") + + info("This diff proves the branch diverged from main with exactly one change.") + + +def show_commit_log(): + """Print the recent commit history of the experiment branch.""" + resp = api( + "GET", + f"/repositories/{REPO_NAME}/refs/{EXP_BRANCH}/commits", + params={"amount": 5}, + ) + commits = resp.json().get("results", []) + ok(f"Last {len(commits)} commit(s) on '{EXP_BRANCH}':") + for c in commits: + cid = c.get("id", "")[:16] + msg = c.get("message", "")[:55] + creator = c.get("committer", "") + print(f" {cid}… {msg}") + if creator: + print(f" {'':18}committer: {creator}") + + +def merge_to_main(): + """Merge the validated experiment branch back to main.""" + info(f"Merging '{EXP_BRANCH}' → '{MAIN_BRANCH}'") + + resp = api( + "POST", + f"/repositories/{REPO_NAME}/refs/{EXP_BRANCH}/merge/{MAIN_BRANCH}", + json={"message": "merge: payment_history_score feature — validated, AUC +0.03"}, + ) + ref = resp.json().get("reference", "") + ok(f"Merge committed → {ref[:16]}…") + + +def verify_main_after_merge(): + """Confirm main now has the new feature and show the final log.""" + resp = api( + "GET", + f"/repositories/{REPO_NAME}/refs/{MAIN_BRANCH}/commits", + params={"amount": 4}, + ) + commits = resp.json().get("results", []) + ok(f"Main branch — last {len(commits)} commits:") + for c in commits: + cid = c.get("id", "")[:16] + msg = c.get("message", "")[:55] + print(f" {cid}… {msg}") + + # Confirm the file is present on main + resp = api( + "HEAD", + f"/repositories/{REPO_NAME}/refs/{MAIN_BRANCH}/objects", + params={"path": "data/customers.csv"}, + ) + ok("data/customers.csv is present on main (with payment_history_score column)") + + +# ───────────────────────────────────────────────────────────────────────────── +# Main +# ───────────────────────────────────────────────────────────────────────────── + +def main(): + TOTAL = 11 + + banner("lakeFS + NetApp ONTAP S3 | Phase 1 Demo") + print(f"\n Use case : Customer Churn — ML Feature Engineering") + print(f" lakeFS : {LAKEFS_HOST}") + print(f" Bucket : s3://{ONTAP_S3_BUCKET} (NetApp ONTAP native S3)") + print(f" Repo : {REPO_NAME}") + + step(1, TOTAL, "Verifying lakeFS connectivity") + check_lakefs() + + step(2, TOTAL, "Creating repository") + create_repository() + + step(3, TOTAL, "Uploading baseline dataset → main branch") + upload_baseline() + + step(4, TOTAL, "Committing baseline to main") + main_commit = commit_to_main() + + step(5, TOTAL, "Creating experiment branch") + create_experiment_branch() + + step(6, TOTAL, "Uploading feature-engineered dataset → experiment branch") + upload_experiment_data() + + step(7, TOTAL, "Committing experiment on branch") + exp_commit = commit_on_branch() + + step(8, TOTAL, "Diff: experiment branch vs main") + show_diff() + + step(9, TOTAL, "Commit history on experiment branch") + show_commit_log() + + step(10, TOTAL, "Merging experiment → main") + merge_to_main() + + step(11, TOTAL, "Verifying main post-merge") + verify_main_after_merge() + + # ── Done ────────────────────────────────────────────────────────────────── + width = 62 + print(f"\n╔{'═' * width}╗") + print(f"║ {'DEMO COMPLETE':^{width - 2}} ║") + print(f"╠{'═' * width}╣") + lines = [ + f"Repository : {REPO_NAME}", + f"Storage : {STORAGE_NS}", + f" : (data physically on NetApp ONTAP S3)", + f"UI : {LAKEFS_HOST}/repositories", + ] + for line in lines: + print(f"║ {line:<{width - 2}}║") + print(f"╚{'═' * width}╝\n") + + +if __name__ == "__main__": + main() diff --git a/01_standalone_examples/netapp-ontap/demo/requirements.txt b/01_standalone_examples/netapp-ontap/demo/requirements.txt new file mode 100644 index 000000000..a8608b2c6 --- /dev/null +++ b/01_standalone_examples/netapp-ontap/demo/requirements.txt @@ -0,0 +1 @@ +requests>=2.28.0 diff --git a/01_standalone_examples/netapp-ontap/demo/seed_data/customers_baseline.csv b/01_standalone_examples/netapp-ontap/demo/seed_data/customers_baseline.csv new file mode 100644 index 000000000..d3aac015c --- /dev/null +++ b/01_standalone_examples/netapp-ontap/demo/seed_data/customers_baseline.csv @@ -0,0 +1,21 @@ +customer_id,tenure_months,monthly_charges,total_charges,contract_type,payment_method,tech_support,internet_service,num_products,churned +C001,12,65.50,786.00,Month-to-month,Electronic check,No,Fiber optic,1,1 +C002,34,89.90,3056.60,One year,Credit card,Yes,Fiber optic,3,0 +C003,2,53.85,107.70,Month-to-month,Electronic check,No,DSL,1,1 +C004,58,99.65,5779.70,Two year,Bank transfer,Yes,Fiber optic,4,0 +C005,1,59.95,59.95,Month-to-month,Electronic check,No,Fiber optic,1,1 +C006,45,75.20,3384.00,One year,Credit card,Yes,DSL,3,0 +C007,8,70.10,560.80,Month-to-month,Mailed check,No,Fiber optic,2,1 +C008,72,109.00,7848.00,Two year,Bank transfer,Yes,Fiber optic,5,0 +C009,6,55.00,330.00,Month-to-month,Electronic check,No,DSL,1,1 +C010,29,85.45,2477.95,One year,Credit card,Yes,Fiber optic,3,0 +C011,3,60.25,180.75,Month-to-month,Electronic check,No,Fiber optic,1,1 +C012,51,95.10,4850.10,Two year,Bank transfer,Yes,Fiber optic,4,0 +C013,15,72.00,1080.00,Month-to-month,Mailed check,No,DSL,2,0 +C014,42,88.55,3719.10,One year,Credit card,Yes,Fiber optic,3,0 +C015,4,58.75,235.00,Month-to-month,Electronic check,No,Fiber optic,1,1 +C016,67,102.30,6854.10,Two year,Bank transfer,Yes,Fiber optic,5,0 +C017,9,63.40,570.60,Month-to-month,Electronic check,No,DSL,1,1 +C018,38,91.20,3465.60,One year,Credit card,Yes,Fiber optic,3,0 +C019,5,57.00,285.00,Month-to-month,Electronic check,No,Fiber optic,1,1 +C020,55,97.80,5379.00,Two year,Bank transfer,Yes,Fiber optic,4,0 diff --git a/01_standalone_examples/netapp-ontap/demo/seed_data/customers_experiment.csv b/01_standalone_examples/netapp-ontap/demo/seed_data/customers_experiment.csv new file mode 100644 index 000000000..8aa98bd68 --- /dev/null +++ b/01_standalone_examples/netapp-ontap/demo/seed_data/customers_experiment.csv @@ -0,0 +1,21 @@ +customer_id,tenure_months,monthly_charges,total_charges,contract_type,payment_method,tech_support,internet_service,num_products,payment_history_score,churned +C001,12,65.50,786.00,Month-to-month,Electronic check,No,Fiber optic,1,0.41,1 +C002,34,89.90,3056.60,One year,Credit card,Yes,Fiber optic,3,0.89,0 +C003,2,53.85,107.70,Month-to-month,Electronic check,No,DSL,1,0.35,1 +C004,58,99.65,5779.70,Two year,Bank transfer,Yes,Fiber optic,4,0.95,0 +C005,1,59.95,59.95,Month-to-month,Electronic check,No,Fiber optic,1,0.28,1 +C006,45,75.20,3384.00,One year,Credit card,Yes,DSL,3,0.91,0 +C007,8,70.10,560.80,Month-to-month,Mailed check,No,Fiber optic,2,0.52,1 +C008,72,109.00,7848.00,Two year,Bank transfer,Yes,Fiber optic,5,0.97,0 +C009,6,55.00,330.00,Month-to-month,Electronic check,No,DSL,1,0.33,1 +C010,29,85.45,2477.95,One year,Credit card,Yes,Fiber optic,3,0.85,0 +C011,3,60.25,180.75,Month-to-month,Electronic check,No,Fiber optic,1,0.30,1 +C012,51,95.10,4850.10,Two year,Bank transfer,Yes,Fiber optic,4,0.93,0 +C013,15,72.00,1080.00,Month-to-month,Mailed check,No,DSL,2,0.68,0 +C014,42,88.55,3719.10,One year,Credit card,Yes,Fiber optic,3,0.88,0 +C015,4,58.75,235.00,Month-to-month,Electronic check,No,Fiber optic,1,0.31,1 +C016,67,102.30,6854.10,Two year,Bank transfer,Yes,Fiber optic,5,0.96,0 +C017,9,63.40,570.60,Month-to-month,Electronic check,No,DSL,1,0.44,1 +C018,38,91.20,3465.60,One year,Credit card,Yes,Fiber optic,3,0.87,0 +C019,5,57.00,285.00,Month-to-month,Electronic check,No,Fiber optic,1,0.29,1 +C020,55,97.80,5379.00,Two year,Bank transfer,Yes,Fiber optic,4,0.94,0 diff --git a/01_standalone_examples/netapp-ontap/docker-compose.yml b/01_standalone_examples/netapp-ontap/docker-compose.yml new file mode 100644 index 000000000..97f11e444 --- /dev/null +++ b/01_standalone_examples/netapp-ontap/docker-compose.yml @@ -0,0 +1,25 @@ +# docker-compose.yml +# Runs PostgreSQL only. lakeFS runs as a native binary on the Mac host +# so it can reach the ONTAP VM on the VMware hostonly network. + +services: + postgres: + image: postgres:15-alpine + container_name: lakefs-postgres + environment: + POSTGRES_USER: ${POSTGRES_USER:-lakefs} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-lakefs} + POSTGRES_DB: ${POSTGRES_DB:-lakefs} + ports: + - "5432:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-lakefs}"] + interval: 5s + timeout: 5s + retries: 10 + restart: unless-stopped + +volumes: + postgres_data: diff --git a/01_standalone_examples/netapp-ontap/lakefs.yaml.example b/01_standalone_examples/netapp-ontap/lakefs.yaml.example new file mode 100644 index 000000000..bc1fd9503 --- /dev/null +++ b/01_standalone_examples/netapp-ontap/lakefs.yaml.example @@ -0,0 +1,47 @@ +# ───────────────────────────────────────────────────────────────────────────── +# lakeFS configuration template for the ONTAP demo. +# Copy this file to lakefs.yaml and fill in all REPLACE_ME values. +# lakefs.yaml is gitignored. +# ───────────────────────────────────────────────────────────────────────────── + +database: + type: postgres + postgres: + connection_string: "postgresql://lakefs:lakefs@localhost:5432/lakefs?sslmode=disable" + +auth: + encrypt: + # Must be >= 32 chars. Generate with: openssl rand -hex 20 + secret_key: "REPLACE_WITH_32_CHAR_RANDOM_STRING" + +installation: + user_name: admin + access_key_id: "AKIAIOSFODNN7EXAMPLE" + secret_access_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + +blockstore: + type: s3 + s3: + # FSx for ONTAP: use the SVM's iSCSI/NFS/S3 endpoint IP (found in FSx console + # under Storage Virtual Machines → Endpoints → Management DNS name). + # Use http:// to avoid certificate issues in the demo. + # Example: http://198.19.255.x (the SVM management IP shown in FSx console) + endpoint: "http://REPLACE_ME_WITH_FSX_SVM_IP" + + # ONTAP S3 does not support virtual-hosted-style URLs. + force_path_style: true + + # ONTAP S3 does not advertise AWS regions; any value works here. + region: "us-east-1" + + credentials: + # Generated by ONTAP CLI: vserver object-store-server user create -user lakefs + access_key_id: "REPLACE_ME" + secret_access_key: "REPLACE_ME" + +server: + listen_address: ":8000" + +logging: + level: "INFO" + format: "text" diff --git a/01_standalone_examples/netapp-ontap/ontap-setup-notes.md b/01_standalone_examples/netapp-ontap/ontap-setup-notes.md new file mode 100644 index 000000000..c74bb062f --- /dev/null +++ b/01_standalone_examples/netapp-ontap/ontap-setup-notes.md @@ -0,0 +1,408 @@ +# ONTAP Simulator Setup Notes +## NetApp ONTAP 9.18.1-CM (vsim) — Phase 1 Demo + +> ⚠️ **DEPRECATED — reference only.** This local-simulator path (UTM/VMware +> Fusion on a Mac) was abandoned: x86 emulation of the ONTAP simulator on Apple +> Silicon is too slow to be usable for a demo. Use the **AWS FSx for NetApp +> ONTAP** path instead — see `README.md` and `SETUP_GUIDE.md`. + +This document contains every command you need to run to get the ONTAP simulator +up and configured for the lakeFS demo. Keep this open while doing the setup. + +--- + +## Part 1 — Install UTM + +> VMware Fusion 13 on Apple Silicon **cannot run x86 VMs** (only ARM64 guests). +> UTM with QEMU emulation is the correct tool for this OVA on an M-series Mac. + +--- + +### Option A — UTM (recommended if Broadcom portal is giving you trouble) + +UTM is free, open source, and installs in one command: + +```bash +brew install --cask utm +brew install qemu # needed for VMDK → QCOW2 conversion +``` + +**Convert the OVA for UTM:** + +```bash +mkdir -p /tmp/ontap-ova +cd /tmp/ontap-ova +tar -xf ~/Downloads/vsim-netapp-DOT9.18.1-cm_nodar.ova + +# Convert all 4 VMDKs to QCOW2 +for vmdk in *.vmdk; do + echo "Converting $vmdk ..." + qemu-img convert -f vmdk -O qcow2 "$vmdk" "${vmdk%.vmdk}.qcow2" +done +ls -lh *.qcow2 +``` + +**Create the VM in UTM:** + +1. UTM → **"+"** → **Emulate** (not Virtualize) +2. OS: Other | Architecture: **x86_64** | System: **Intel ICH9 based PC (Q35)** +3. RAM: **6144 MB** | CPU: **2** | Skip default storage disk +4. Name: `ONTAP-sim-9.18.1` → Save (don't start yet) +5. Edit VM → **Drives** tab → delete any auto-created disk +6. Add each qcow2 file in order (disk1 through disk4): + - New Drive → Import → select `.qcow2` | Interface: **IDE** +7. Edit VM → **Network** tab: + - Network Mode: **Host Only** (not Shared Network — Host Only lets your Mac reach the VM) + - Host Network: **Default (private)** + - Emulated card: **Intel Gigabit Ethernet (e1000)** + +**UTM network addresses (use these in the cluster wizard):** + +``` +Node mgmt LIF: 10.0.2.10 +Cluster mgmt LIF: 10.0.2.11 +Default gateway: 10.0.2.2 ← UTM's virtual gateway +DNS: 8.8.8.8 +S3 data LIF: 10.0.2.20 ← use in lakeFS config +``` + +> **Performance note:** UTM runs x86 via QEMU emulation on Apple Silicon. +> First boot takes 10–15 min. S3 API calls will be ~3× slower than native. +> Fine for a demo — just allow more time. + +--- + +### Option B — VMware Fusion Pro (best performance, more complex to download) + +VMware Fusion Pro is free since May 2024 but the Broadcom portal is awkward. + +**If the downloads page is empty, try these fixes in order:** + +1. After logging in, go to **My Dashboard → My Downloads**. If VMware Fusion + is not listed, search the product catalogue for "VMware Fusion", open the + product page, and click **"Register"** or **"Add to my account"**, then + return to downloads. + +2. Navigate via: `support.broadcom.com` → top menu **Software** → search + "VMware Fusion" → product page → **Go to downloads** tab. + +3. Try the **Mac App Store** — search "VMware Fusion". Same product, no + Broadcom account needed. + +**Once installed:** + +1. File → Import → select `~/Downloads/vsim-netapp-DOT9.18.1-cm_nodar.ova` +2. Name it `ONTAP-sim-9.18.1`, click Continue through compatibility warning +3. VM Settings → Network Adapter → **Host-only** (vmnet1) +4. Check the hostonly subnet: VMware Fusion → Preferences → Network → vmnet1 + +**VMware Fusion hostonly network addresses:** + +``` +Mac gateway (vmnet1): typically 192.168.233.1 +Node mgmt LIF: 192.168.233.10 +Cluster mgmt LIF: 192.168.233.11 +S3 data LIF: 192.168.233.20 ← use in lakeFS config +``` + +> Substitute your actual vmnet1 subnet if it differs from 192.168.233.x. + +--- + +## Part 2 — Import and Boot the OVA + +### Import + +1. Open VMware Fusion +2. File → Import… +3. Select: `~/Downloads/vsim-netapp-DOT9.18.1-cm_nodar.ova` +4. Name the VM: `ONTAP-sim-9.18.1` +5. VMware will warn about compatibility — click **Continue** +6. Save location: default is fine (~5 GB disk) + +### Network adapter settings (CRITICAL) + +The OVA comes with **hostonly** networking. You need the VM on a network that +your Mac host can reach. VMware Fusion hostonly is fine — keep it. + +1. Before starting the VM: VM → Settings → Network Adapter +2. Confirm it is set to **Host-only** (vmnet1) +3. Note the hostonly subnet (usually shown as `192.168.x.0/24` or `172.16.x.0/24`) + - You can check: VMware Fusion menu → Preferences → Network + +The Mac host can always reach the hostonly network. Write down: + +``` +VMware Fusion hostonly subnet: 192.168.___.0/24 +VMware Fusion gateway (Mac): 192.168.___.1 ← this is the Mac-side IP +``` + +### First boot and cluster wizard + +1. Start the VM +2. Wait ~2 minutes for the ONTAP boot sequence (lots of output) +3. At the `cluster setup` prompt, follow the wizard: + +``` +Welcome to the cluster setup wizard. +... +Enter the cluster name: ontap-sim +Create a new cluster or join an existing cluster? [create/join]: create +Do you want to use IPv6? [n]: n +... +Enter the node management interface details + IP address: 192.168.233.10 ← use an IP in your hostonly subnet + Netmask: 255.255.255.0 + Default gateway: 192.168.233.1 ← the Mac-side IP of vmnet1 +... +Enter the cluster management interface details + IP address: 192.168.233.11 ← one more IP in the same subnet + Netmask: 255.255.255.0 + Default gateway: 192.168.233.1 +... +Enter DNS domain names []: lab.local +Enter DNS server IP addresses []: 8.8.8.8 +... +Enter the cluster base license key: [leave blank for simulator — no license needed] +... +Enter new admin password: NetApp1! +Retype new admin password: NetApp1! +``` + +> **Note on IPs:** Replace `192.168.233.x` with addresses in YOUR hostonly +> subnet. The first host address (`.1`) is the Mac gateway. Use `.10` for node +> mgmt, `.11` for cluster mgmt, `.20` for the S3 data LIF. + +4. Wait for cluster setup to complete (~3-5 minutes) +5. The shell prompt `ontap-sim::>` appears when ready + +### Verify access from Mac + +From your Mac terminal: +```bash +# Ping the node mgmt IP you configured +ping -c 3 192.168.233.10 + +# SSH in (accept fingerprint, password is what you set above) +ssh admin@192.168.233.11 +``` + +--- + +## Part 3 — Configure ONTAP S3 (CLI commands) + +All commands below are run **inside the ONTAP CLI** (either via SSH or the VM console). + +SSH in: +```bash +ssh admin@192.168.233.11 # use your cluster mgmt IP +# password: NetApp1! (or whatever you set) +``` + +### 3.1 — Survey the cluster + +```ontap +# Check node name (you'll need it for LIF creation) +node show + +# Check available aggregates (you need one for the SVM root volume) +storage aggregate show + +# Check existing SVMs +vserver show +``` + +Expected: one node named `ontap-sim-01`, one or two aggregates (`aggr0`, `aggr1`). +Use `aggr1` for data (aggr0 is reserved for system). + +### 3.2 — Create the S3 SVM + +```ontap +# Create a new SVM dedicated to S3 +vserver create -vserver s3svm -subtype default + +# Assign S3 as the allowed protocol +vserver modify -vserver s3svm -allowed-protocols s3 + +# Verify +vserver show -vserver s3svm +``` + +### 3.3 — Create a data network interface (LIF) for S3 + +The data LIF is the IP address that lakeFS will use as the S3 endpoint. +Use an IP in your hostonly subnet (e.g. `.20`). + +```ontap +network interface create \ + -vserver s3svm \ + -lif s3_data_1 \ + -service-policy data-s3-server \ + -home-node ontap-sim-01 \ + -home-port e0c \ + -address 192.168.233.20 \ + -netmask 255.255.255.0 \ + -status-admin up + +# Verify the LIF is up +network interface show -vserver s3svm +``` + +> Replace `ontap-sim-01` with the actual node name from step 3.1. +> Replace `e0c` with an available port (check: `network port show -node ontap-sim-01`). +> Replace `192.168.233.20` with an available IP in your hostonly subnet. + +### 3.4 — Enable S3 server on the SVM + +Using HTTP (port 80) avoids all certificate complications for the demo. + +```ontap +vserver object-store-server create \ + -vserver s3svm \ + -object-store-server s3.ontap.local \ + -is-http-enabled true \ + -is-https-enabled false \ + -port 80 + +# Verify +vserver object-store-server show +``` + +### 3.5 — Create the S3 user for lakeFS + +```ontap +vserver object-store-server user create \ + -vserver s3svm \ + -user lakefs +``` + +**CRITICAL: The output shows the access key and secret key exactly once.** +Copy them immediately: + +``` +Access Key: +Secret Key: +``` + +Paste these into your `.env` file: +``` +ONTAP_S3_ACCESS_KEY= +ONTAP_S3_SECRET_KEY= +``` + +And into `lakefs.yaml` under `blockstore.s3.credentials`. + +If you lose the keys, recreate the user: +```ontap +vserver object-store-server user delete -vserver s3svm -user lakefs +vserver object-store-server user create -vserver s3svm -user lakefs +``` + +### 3.6 — Create the S3 bucket + +```ontap +vserver object-store-server bucket create \ + -vserver s3svm \ + -bucket lakefs-data \ + -size 50GB + +# Verify +vserver object-store-server bucket show +``` + +### 3.7 — Grant the lakefs user full access to the bucket + +```ontap +vserver object-store-server bucket policy statement create \ + -vserver s3svm \ + -bucket lakefs-data \ + -effect allow \ + -action * \ + -principal lakefs \ + -resource lakefs-data,lakefs-data/* + +# Verify policy +vserver object-store-server bucket policy statement show -vserver s3svm -bucket lakefs-data +``` + +### 3.8 — Verify everything from the ONTAP side + +```ontap +# S3 server running? +vserver object-store-server show + +# LIF is up? +network interface show -vserver s3svm + +# Bucket exists? +vserver object-store-server bucket show + +# User and keys? +vserver object-store-server user show +``` + +--- + +## Part 4 — Verify from Mac host + +From your Mac terminal (NOT inside ONTAP): + +```bash +# Load .env values +source .env + +# Basic reachability +ping -c 3 192.168.233.20 # replace with your data LIF IP + +# HTTP check — should return 200, 301, or 403 +curl -v "http://192.168.233.20/lakefs-data/" + +# Full S3 credential test (requires awscli: brew install awscli) +AWS_ACCESS_KEY_ID=$ONTAP_S3_ACCESS_KEY \ +AWS_SECRET_ACCESS_KEY=$ONTAP_S3_SECRET_KEY \ +aws s3 ls s3://lakefs-data/ \ + --endpoint-url "http://192.168.233.20" \ + --region us-east-1 +``` + +If `aws s3 ls` returns empty output (no error) the bucket is accessible. + +--- + +## Quick-reference: IP addresses used in this demo + +### If using UTM (Shared Network) + +| Role | IP | Notes | +|---------------------------|------------------|-------------------------------| +| UTM virtual gateway | 192.168.64.1 | Set by UTM/QEMU | +| ONTAP node mgmt LIF | 192.168.64.10 | Set during cluster wizard | +| ONTAP cluster mgmt LIF | 192.168.64.11 | SSH target | +| ONTAP S3 data LIF | 192.168.64.20 | lakeFS S3 endpoint | + +### If using VMware Fusion (Host-only vmnet1) + +| Role | IP | Notes | +|---------------------------|------------------|-------------------------------| +| Mac (vmnet1 gateway) | 192.168.233.1 | Set by VMware Fusion | +| ONTAP node mgmt LIF | 192.168.233.10 | Set during cluster wizard | +| ONTAP cluster mgmt LIF | 192.168.233.11 | SSH target | +| ONTAP S3 data LIF | 192.168.233.20 | lakeFS S3 endpoint | + +> All IPs are examples. Substitute your actual subnet throughout. +> After you know your data LIF IP, update `.env` (ONTAP_S3_ENDPOINT) and +> `lakefs.yaml` (blockstore.s3.endpoint) accordingly. + +--- + +## Troubleshooting ONTAP + +| Symptom | Fix | +|---|---| +| `ping` fails to VM | Check VM is running, VMware hostonly network is enabled | +| SSH refused | Try VM console; ensure cluster setup completed | +| `data-s3-server` service policy unknown | ONTAP < 9.8; update or use `-role data -data-protocol` syntax | +| LIF fails to come up | Check port is available: `network port show` | +| HTTP 403 from S3 | Bucket policy missing; re-run step 3.7 | +| Access key lost | Delete user and recreate (step 3.5) | +| Simulator very slow | Normal on x86 emulation; allow 5-10 min on first boot | diff --git a/01_standalone_examples/netapp-ontap/scripts/check-prereqs.sh b/01_standalone_examples/netapp-ontap/scripts/check-prereqs.sh new file mode 100755 index 000000000..4f05c15e6 --- /dev/null +++ b/01_standalone_examples/netapp-ontap/scripts/check-prereqs.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash +# check-prereqs.sh +# Verifies that every tool needed for the demo is present. +# Run this first before anything else. + +set -euo pipefail + +PASS=0 +FAIL=0 + +ok() { echo " [OK] $*"; PASS=$((PASS+1)); } +fail() { echo " [!!] $*"; FAIL=$((FAIL+1)); } +info() { echo " $*"; } + +echo "" +echo "╔══════════════════════════════════════════════════════╗" +echo "║ lakeFS + NetApp ONTAP Demo — Prerequisites Check ║" +echo "╚══════════════════════════════════════════════════════╝" +echo "" + +# ── Docker ──────────────────────────────────────────────────────────────────── +echo "── Docker ───────────────────────────────────────────────" +if docker --version &>/dev/null; then + ok "Docker: $(docker --version)" +else + fail "Docker not found. Install Docker Desktop from https://docs.docker.com/desktop/install/mac-install/" +fi + +if docker compose version &>/dev/null; then + ok "Docker Compose: $(docker compose version)" +else + fail "Docker Compose not found (requires Docker Desktop >= 4.x)" +fi + +if docker info &>/dev/null; then + ok "Docker daemon is running" +else + fail "Docker daemon is not running — open Docker Desktop" +fi +echo "" + +# ── VMware Fusion ───────────────────────────────────────────────────────────── +echo "── VMware Fusion (required to run the ONTAP OVA) ────────" +if [ -d "/Applications/VMware Fusion.app" ]; then + ok "VMware Fusion is installed" +else + fail "VMware Fusion not found" + info "Install VMware Fusion Pro (free) from:" + info " https://support.broadcom.com/group/ecx/productdownloads?subfamily=VMware+Fusion" + info " Sign in with a free Broadcom account, then download 'VMware Fusion Pro'." + info " Alternatively: brew install --cask vmware-fusion (may be disabled; use direct download)" +fi +echo "" + +# ── Python ──────────────────────────────────────────────────────────────────── +echo "── Python ───────────────────────────────────────────────" +if python3 --version &>/dev/null; then + PYVER=$(python3 --version) + ok "Python: $PYVER" +else + fail "python3 not found — install via: brew install python" +fi + +if python3 -c "import requests" 2>/dev/null; then + ok "requests library available" +else + fail "requests not installed — run: pip3 install requests" +fi + +if python3 -c "import lakefs" 2>/dev/null; then + ok "lakefs SDK available: $(python3 -c 'import lakefs; print(lakefs.__version__)')" +else + info "lakefs SDK not installed (optional, demo uses requests directly)" +fi +echo "" + +# ── lakeFS binary ───────────────────────────────────────────────────────────── +echo "── lakeFS binary ────────────────────────────────────────" +LAKEFS_BIN="" +if command -v lakefs &>/dev/null; then + LAKEFS_BIN="lakefs" + ok "lakefs in PATH: $(lakefs --version)" +elif [ -f "$(dirname "$0")/../bin/lakefs" ]; then + LAKEFS_BIN="$(dirname "$0")/../bin/lakefs" + ok "lakefs binary found at ./bin/lakefs: $($LAKEFS_BIN --version)" +else + fail "lakefs binary not found" + info "Download with scripts/install-lakefs.sh (or manually):" + info " VERSION=1.80.0" + info " curl -L https://github.com/treeverse/lakeFS/releases/download/v\$VERSION/lakeFS_\${VERSION}_Darwin_arm64.tar.gz \\" + info " | tar -xz -C ./bin" +fi +echo "" + +# ── Config files ────────────────────────────────────────────────────────────── +echo "── Config files ─────────────────────────────────────────" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(dirname "$SCRIPT_DIR")" + +if [ -f "$ROOT/.env" ]; then + ok ".env exists" + # Check for unfilled placeholders + if grep -q "REPLACE_ME" "$ROOT/.env" 2>/dev/null; then + fail ".env still contains REPLACE_ME — fill in your ONTAP credentials" + fi +else + fail ".env not found — copy .env.example to .env and fill in values" +fi + +if [ -f "$ROOT/lakefs.yaml" ]; then + ok "lakefs.yaml exists" + if grep -q "REPLACE_ME" "$ROOT/lakefs.yaml" 2>/dev/null; then + fail "lakefs.yaml still contains REPLACE_ME — fill in your values" + fi +else + fail "lakefs.yaml not found — copy lakefs.yaml.example to lakefs.yaml and fill in values" +fi +echo "" + +# ── OVA file ────────────────────────────────────────────────────────────────── +echo "── ONTAP Simulator OVA ──────────────────────────────────" +OVA="$HOME/Downloads/vsim-netapp-DOT9.18.1-cm_nodar.ova" +if [ -f "$OVA" ]; then + SIZE=$(du -sh "$OVA" | cut -f1) + ok "OVA found: $OVA ($SIZE)" +else + fail "OVA not found at $OVA" + info "Expected: vsim-netapp-DOT9.18.1-cm_nodar.ova" +fi +echo "" + +# ── Summary ─────────────────────────────────────────────────────────────────── +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " PASSED: $PASS FAILED: $FAIL" +if [ "$FAIL" -eq 0 ]; then + echo " All prerequisites met. You are ready to proceed." +else + echo " Fix the items above, then re-run this script." +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "" diff --git a/01_standalone_examples/netapp-ontap/scripts/install-lakefs.sh b/01_standalone_examples/netapp-ontap/scripts/install-lakefs.sh new file mode 100755 index 000000000..269c46c87 --- /dev/null +++ b/01_standalone_examples/netapp-ontap/scripts/install-lakefs.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# install-lakefs.sh +# Downloads the lakeFS macOS arm64 binary to ./bin/lakefs. +# Run once from the project root directory. + +set -euo pipefail + +VERSION="1.80.0" +ARCH="arm64" +OS="Darwin" +DEST="$(cd "$(dirname "$0")/.." && pwd)/bin" +TARBALL="lakeFS_${VERSION}_${OS}_${ARCH}.tar.gz" +URL="https://github.com/treeverse/lakeFS/releases/download/v${VERSION}/${TARBALL}" + +echo "Downloading lakeFS v${VERSION} for ${OS}/${ARCH}..." +echo "From: $URL" +echo "To: $DEST/lakefs" +echo "" + +mkdir -p "$DEST" +curl -L --progress-bar "$URL" | tar -xz -C "$DEST" lakefs + +chmod +x "$DEST/lakefs" +echo "Done. Verify:" +"$DEST/lakefs" --version +echo "" +echo "Add to PATH for this session:" +echo " export PATH=\"$DEST:\$PATH\"" diff --git a/01_standalone_examples/netapp-ontap/scripts/setup-demo.sh b/01_standalone_examples/netapp-ontap/scripts/setup-demo.sh new file mode 100644 index 000000000..5736345d3 --- /dev/null +++ b/01_standalone_examples/netapp-ontap/scripts/setup-demo.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# ───────────────────────────────────────────────────────────────────────────── +# setup-demo.sh — Run this on EC2 after Terraform provisions the infrastructure +# Usage: bash setup-demo.sh +# ───────────────────────────────────────────────────────────────────────────── + +set -e + +SVM_IP="${1:?Usage: $0 }" +ONTAP_ACCESS_KEY="${2:?missing ONTAP_ACCESS_KEY}" +ONTAP_SECRET_KEY="${3:?missing ONTAP_SECRET_KEY}" + +echo "==> Installing dependencies..." +sudo apt-get update -qq +sudo apt-get install -y -qq docker.io python3-pip + +sudo systemctl start docker +sudo usermod -aG docker ubuntu + +echo "==> Starting Postgres..." +sudo docker run -d --name postgres \ + -e POSTGRES_USER=lakefs \ + -e POSTGRES_PASSWORD=lakefs \ + -e POSTGRES_DB=lakefs \ + -p 5432:5432 \ + --restart unless-stopped \ + postgres:15-alpine + +echo "==> Installing lakeFS..." +LAKEFS_VERSION=$(curl -s https://api.github.com/repos/treeverse/lakeFS/releases/latest \ + | grep '"tag_name"' | cut -d'"' -f4 | sed 's/v//') +curl -sL "https://github.com/treeverse/lakeFS/releases/download/v${LAKEFS_VERSION}/lakeFS_${LAKEFS_VERSION}_Linux_x86_64.tar.gz" | tar xz +sudo mv lakefs /usr/local/bin/ + +echo "==> Writing lakeFS config..." +cat > ~/lakefs.yaml << EOF +database: + type: postgres + postgres: + connection_string: "postgresql://lakefs:lakefs@localhost:5432/lakefs?sslmode=disable" + +auth: + encrypt: + secret_key: "a7f3d9e2b8c4f1a6e5d0b3c7f2a9e4d1b6c8f3a2" + +installation: + user_name: admin + access_key_id: "AKIAIOSFODNN7EXAMPLE" + secret_access_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + +blockstore: + type: s3 + s3: + endpoint: "http://${SVM_IP}" + force_path_style: true + region: "us-east-1" + credentials: + access_key_id: "${ONTAP_ACCESS_KEY}" + secret_access_key: "${ONTAP_SECRET_KEY}" + +logging: + level: "INFO" + format: "text" +EOF + +echo "==> Installing Python demo dependencies..." +pip3 install -q requests + +echo "==> Starting lakeFS..." +nohup lakefs run --config ~/lakefs.yaml > ~/lakefs.log 2>&1 & +sleep 3 + +echo "" +echo "✓ Setup complete!" +echo " lakeFS UI: http://$(curl -s checkip.amazonaws.com):8000" +echo " Complete setup at: http://$(curl -s checkip.amazonaws.com):8000/setup" +echo "" +echo " lakeFS admin credentials (set during /setup):" +echo " Save the Access Key ID and Secret Key shown in the browser." diff --git a/01_standalone_examples/netapp-ontap/scripts/start-services.sh b/01_standalone_examples/netapp-ontap/scripts/start-services.sh new file mode 100755 index 000000000..ebd73ce00 --- /dev/null +++ b/01_standalone_examples/netapp-ontap/scripts/start-services.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# start-services.sh +# Starts PostgreSQL (Docker) and lakeFS (native binary). +# Run from the project root directory. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(dirname "$SCRIPT_DIR")" +PID_FILE="$ROOT/.lakefs.pid" + +cd "$ROOT" + +# ── Sanity checks ───────────────────────────────────────────────────────────── +if [ ! -f ".env" ]; then + echo "ERROR: .env not found. Copy .env.example to .env and fill in values." + exit 1 +fi +if [ ! -f "lakefs.yaml" ]; then + echo "ERROR: lakefs.yaml not found. Copy lakefs.yaml.example to lakefs.yaml and fill in values." + exit 1 +fi + +# Load env vars for Postgres settings +set -a; source .env; set +a + +# ── Locate lakeFS binary ────────────────────────────────────────────────────── +if command -v lakefs &>/dev/null; then + LAKEFS_BIN="lakefs" +elif [ -f "./bin/lakefs" ]; then + LAKEFS_BIN="./bin/lakefs" + export PATH="$ROOT/bin:$PATH" +else + echo "ERROR: lakeFS binary not found." + echo "Run: bash scripts/install-lakefs.sh" + exit 1 +fi + +echo "" +echo "Starting lakeFS + NetApp ONTAP demo environment" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +# ── Step 1: Start PostgreSQL ────────────────────────────────────────────────── +echo "" +echo "[1/3] Starting PostgreSQL..." +docker compose up -d postgres + +echo " Waiting for PostgreSQL to accept connections..." +for i in $(seq 1 30); do + if docker compose exec -T postgres pg_isready -U "${POSTGRES_USER:-lakefs}" -q 2>/dev/null; then + echo " PostgreSQL is ready." + break + fi + if [ "$i" -eq 30 ]; then + echo "ERROR: PostgreSQL did not become ready in time." + exit 1 + fi + sleep 1 +done + +# ── Step 2: Run lakeFS migrations ───────────────────────────────────────────── +echo "" +echo "[2/3] Running lakeFS database migrations..." +$LAKEFS_BIN migrate up --config lakefs.yaml +echo " Migrations complete." + +# ── Step 3: Start lakeFS ────────────────────────────────────────────────────── +echo "" +echo "[3/3] Starting lakeFS..." + +if [ -f "$PID_FILE" ]; then + OLD_PID=$(cat "$PID_FILE") + if kill -0 "$OLD_PID" 2>/dev/null; then + echo " lakeFS already running (PID $OLD_PID). Stop it first with: bash scripts/stop-services.sh" + exit 0 + else + rm -f "$PID_FILE" + fi +fi + +$LAKEFS_BIN run --config lakefs.yaml > "$ROOT/lakefs.log" 2>&1 & +LAKEFS_PID=$! +echo $LAKEFS_PID > "$PID_FILE" + +echo " lakeFS starting (PID $LAKEFS_PID)..." +sleep 3 + +# Verify it started +if ! kill -0 "$LAKEFS_PID" 2>/dev/null; then + echo "ERROR: lakeFS failed to start. Check logs:" + tail -20 "$ROOT/lakefs.log" + exit 1 +fi + +echo "" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " Services running:" +echo " PostgreSQL → localhost:5432" +echo " lakeFS → http://localhost:8000 (PID $LAKEFS_PID)" +echo "" +echo " lakeFS UI: http://localhost:8000" +echo " Logs: tail -f lakefs.log" +echo " Stop: bash scripts/stop-services.sh" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "" +echo "Next: run bash scripts/verify-connectivity.sh to confirm ONTAP S3 is reachable." +echo "" diff --git a/01_standalone_examples/netapp-ontap/scripts/stop-services.sh b/01_standalone_examples/netapp-ontap/scripts/stop-services.sh new file mode 100755 index 000000000..8e9908c9c --- /dev/null +++ b/01_standalone_examples/netapp-ontap/scripts/stop-services.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# stop-services.sh +# Stops lakeFS and PostgreSQL. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(dirname "$SCRIPT_DIR")" +PID_FILE="$ROOT/.lakefs.pid" + +cd "$ROOT" + +echo "Stopping services..." + +if [ -f "$PID_FILE" ]; then + PID=$(cat "$PID_FILE") + if kill -0 "$PID" 2>/dev/null; then + kill "$PID" + echo " lakeFS stopped (was PID $PID)" + else + echo " lakeFS was not running" + fi + rm -f "$PID_FILE" +else + echo " No .lakefs.pid file found — lakeFS may not be running" +fi + +docker compose down +echo " PostgreSQL stopped" +echo "Done." diff --git a/01_standalone_examples/netapp-ontap/scripts/verify-connectivity.sh b/01_standalone_examples/netapp-ontap/scripts/verify-connectivity.sh new file mode 100755 index 000000000..ad3f3fcb9 --- /dev/null +++ b/01_standalone_examples/netapp-ontap/scripts/verify-connectivity.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +# verify-connectivity.sh +# Tests that the Mac can reach the ONTAP S3 endpoint AND that lakeFS is up. +# Run after start-services.sh and after the ONTAP VM is booted and configured. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(dirname "$SCRIPT_DIR")" + +cd "$ROOT" + +if [ ! -f ".env" ]; then + echo "ERROR: .env not found." + exit 1 +fi + +set -a; source .env; set +a + +PASS=0; FAIL=0 + +ok() { echo " [OK] $*"; PASS=$((PASS+1)); } +fail() { echo " [!!] $*"; FAIL=$((FAIL+1)); } +info() { echo " $*"; } + +echo "" +echo "Verifying connectivity..." +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +# ── 1. ONTAP S3 endpoint reachable? ────────────────────────────────────────── +echo "" +echo "── ONTAP S3 ─────────────────────────────────────────────" +info "Endpoint: $ONTAP_S3_ENDPOINT" + +HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" \ + --max-time 5 \ + "$ONTAP_S3_ENDPOINT" 2>/dev/null || echo "000") + +if [ "$HTTP_CODE" = "000" ]; then + fail "Cannot reach $ONTAP_S3_ENDPOINT (connection refused or timed out)" + info "Check: Is the ONTAP VM running? Is the S3 server enabled on the SVM?" + info "Check: Is the data LIF IP correct? Try: ping $(echo $ONTAP_S3_ENDPOINT | sed 's|http[s]*://||' | cut -d: -f1)" +elif [ "$HTTP_CODE" = "200" ] || [ "$HTTP_CODE" = "301" ] || [ "$HTTP_CODE" = "403" ] || [ "$HTTP_CODE" = "405" ]; then + ok "ONTAP S3 endpoint responded (HTTP $HTTP_CODE)" +else + fail "Unexpected HTTP $HTTP_CODE from ONTAP S3 endpoint" +fi + +# ── 2. ONTAP S3 bucket accessible? ─────────────────────────────────────────── +echo "" +info "Testing bucket access with AWS CLI (if installed)..." +if command -v aws &>/dev/null; then + if aws s3 ls "s3://$ONTAP_S3_BUCKET/" \ + --endpoint-url "$ONTAP_S3_ENDPOINT" \ + --access-key-id "${ONTAP_S3_ACCESS_KEY:-x}" \ + --secret-access-key "${ONTAP_S3_SECRET_KEY:-x}" \ + --no-sign-request 2>/dev/null || \ + AWS_ACCESS_KEY_ID="$ONTAP_S3_ACCESS_KEY" \ + AWS_SECRET_ACCESS_KEY="$ONTAP_S3_SECRET_KEY" \ + aws s3 ls "s3://$ONTAP_S3_BUCKET/" \ + --endpoint-url "$ONTAP_S3_ENDPOINT" \ + --region us-east-1 2>/dev/null; then + ok "Bucket '$ONTAP_S3_BUCKET' is accessible via AWS CLI" + else + fail "Cannot list bucket '$ONTAP_S3_BUCKET' — check credentials and bucket policy" + fi +else + info "AWS CLI not installed — skipping bucket list test" + info "Install with: brew install awscli" +fi + +# ── 3. lakeFS health ────────────────────────────────────────────────────────── +echo "" +echo "── lakeFS ───────────────────────────────────────────────" +LAKEFS_HOST="${LAKEFS_HOST:-http://localhost:8000}" +info "Host: $LAKEFS_HOST" + +HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" \ + --max-time 5 \ + "$LAKEFS_HOST/api/v1/healthcheck" 2>/dev/null || echo "000") + +if [ "$HTTP_CODE" = "204" ] || [ "$HTTP_CODE" = "200" ]; then + ok "lakeFS healthcheck passed (HTTP $HTTP_CODE)" +else + fail "lakeFS healthcheck failed (HTTP $HTTP_CODE)" + info "Is lakeFS running? Check: bash scripts/start-services.sh" + info "Check logs: tail -50 lakefs.log" +fi + +# ── 4. lakeFS storage check ─────────────────────────────────────────────────── +HTTP_CODE=$(curl -s -o /tmp/lakefs_storage_check.json -w "%{http_code}" \ + --max-time 5 \ + -u "${LAKEFS_ACCESS_KEY_ID:-AKIAIOSFODNN7EXAMPLE}:${LAKEFS_SECRET_ACCESS_KEY:-wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY}" \ + "$LAKEFS_HOST/api/v1/config" 2>/dev/null || echo "000") + +if [ "$HTTP_CODE" = "200" ]; then + STORAGE_TYPE=$(python3 -c "import json,sys; d=json.load(open('/tmp/lakefs_storage_check.json')); print(d.get('storage_config',{}).get('blockstore_type','?'))" 2>/dev/null || echo "unknown") + ok "lakeFS config endpoint reachable — blockstore type: $STORAGE_TYPE" +elif [ "$HTTP_CODE" = "401" ]; then + fail "lakeFS auth failed (HTTP 401) — check LAKEFS_ACCESS_KEY_ID / LAKEFS_SECRET_ACCESS_KEY in .env" +else + fail "lakeFS config endpoint returned HTTP $HTTP_CODE" +fi + +# ── Summary ─────────────────────────────────────────────────────────────────── +echo "" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " PASSED: $PASS FAILED: $FAIL" +if [ "$FAIL" -eq 0 ]; then + echo " All connectivity checks passed. Run the demo:" + echo " cd demo && python3 demo_flow.py" +else + echo " Fix the issues above before running the demo." + echo " See README.md — Troubleshooting section." +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "" diff --git a/01_standalone_examples/netapp-ontap/terraform/.terraform.lock.hcl b/01_standalone_examples/netapp-ontap/terraform/.terraform.lock.hcl new file mode 100644 index 000000000..1651c9120 --- /dev/null +++ b/01_standalone_examples/netapp-ontap/terraform/.terraform.lock.hcl @@ -0,0 +1,44 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.100.0" + constraints = "~> 5.0" + hashes = [ + "h1:Ijt7pOlB7Tr7maGQIqtsLFbl7pSMIj06TVdkoSBcYOw=", + "zh:054b8dd49f0549c9a7cc27d159e45327b7b65cf404da5e5a20da154b90b8a644", + "zh:0b97bf8d5e03d15d83cc40b0530a1f84b459354939ba6f135a0086c20ebbe6b2", + "zh:1589a2266af699cbd5d80737a0fe02e54ec9cf2ca54e7e00ac51c7359056f274", + "zh:6330766f1d85f01ae6ea90d1b214b8b74cc8c1badc4696b165b36ddd4cc15f7b", + "zh:7c8c2e30d8e55291b86fcb64bdf6c25489d538688545eb48fd74ad622e5d3862", + "zh:99b1003bd9bd32ee323544da897148f46a527f622dc3971af63ea3e251596342", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9f8b909d3ec50ade83c8062290378b1ec553edef6a447c56dadc01a99f4eaa93", + "zh:aaef921ff9aabaf8b1869a86d692ebd24fbd4e12c21205034bb679b9caf883a2", + "zh:ac882313207aba00dd5a76dbd572a0ddc818bb9cbf5c9d61b28fe30efaec951e", + "zh:bb64e8aff37becab373a1a0cc1080990785304141af42ed6aa3dd4913b000421", + "zh:dfe495f6621df5540d9c92ad40b8067376350b005c637ea6efac5dc15028add4", + "zh:f0ddf0eaf052766cfe09dea8200a946519f653c384ab4336e2a4a64fdd6310e9", + "zh:f1b7e684f4c7ae1eed272b6de7d2049bb87a0275cb04dbb7cda6636f600699c9", + "zh:ff461571e3f233699bf690db319dfe46aec75e58726636a0d97dd9ac6e32fb70", + ] +} + +provider "registry.terraform.io/hashicorp/http" { + version = "3.5.0" + hashes = [ + "h1:dl73+8wzQR++HFGoJgDqY3mj3pm14HUuH/CekVyOj5s=", + "zh:047c5b4920751b13425efe0d011b3a23a3be97d02d9c0e3c60985521c9c456b7", + "zh:157866f700470207561f6d032d344916b82268ecd0cf8174fb11c0674c8d0736", + "zh:1973eb9383b0d83dd4fd5e662f0f16de837d072b64a6b7cd703410d730499476", + "zh:212f833a4e6d020840672f6f88273d62a564f44acb0c857b5961cdb3bbc14c90", + "zh:2c8034bc039fffaa1d4965ca02a8c6d57301e5fa9fff4773e684b46e3f78e76a", + "zh:5df353fc5b2dd31577def9cc1a4ebf0c9a9c2699d223c6b02087a3089c74a1c6", + "zh:672083810d4185076c81b16ad13d1224b9e6ea7f4850951d2ab8d30fa6e41f08", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7b4200f18abdbe39904b03537e1a78f21ebafe60f1c861a44387d314fda69da6", + "zh:843feacacd86baed820f81a6c9f7bd32cf302db3d7a0f39e87976ebc7a7cc2ee", + "zh:a9ea5096ab91aab260b22e4251c05f08dad2ed77e43e5e4fadcdfd87f2c78926", + "zh:d02b288922811739059e90184c7f76d45d07d3a77cc48d0b15fd3db14e928623", + ] +} diff --git a/01_standalone_examples/netapp-ontap/terraform/main.tf b/01_standalone_examples/netapp-ontap/terraform/main.tf new file mode 100644 index 000000000..a7b644432 --- /dev/null +++ b/01_standalone_examples/netapp-ontap/terraform/main.tf @@ -0,0 +1,198 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } +} + +provider "aws" { + region = var.aws_region +} + +# ─── Data sources ──────────────────────────────────────────────────────────── + +data "aws_ami" "ubuntu" { + most_recent = true + owners = ["099720109477"] # Canonical + filter { + name = "name" + values = ["ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-*"] + } + filter { + name = "virtualization-type" + values = ["hvm"] + } +} + +data "http" "my_ip" { + url = "https://checkip.amazonaws.com" +} + +locals { + my_ip = "${chomp(data.http.my_ip.response_body)}/32" +} + +# ─── Security Groups ───────────────────────────────────────────────────────── + +resource "aws_security_group" "ec2" { + name = "lakefs-ontap-demo-sg" + description = "lakeFS demo EC2 security group" + vpc_id = var.vpc_id + + ingress { + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = [local.my_ip] + description = "SSH from my IP" + } + + ingress { + from_port = 8000 + to_port = 8000 + protocol = "tcp" + cidr_blocks = [local.my_ip] + description = "lakeFS UI" + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { Name = "lakefs-ontap-demo-sg" } +} + +resource "aws_security_group" "fsx" { + name = "lakefs-ontap-fsx-sg" + description = "FSx for ONTAP security group" + vpc_id = var.vpc_id + + ingress { + from_port = 0 + to_port = 0 + protocol = "-1" + security_groups = [aws_security_group.ec2.id] + description = "All traffic from EC2" + } + + ingress { + from_port = 80 + to_port = 80 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + description = "ONTAP S3 HTTP via NLB for pre-signed URLs" + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { Name = "lakefs-ontap-fsx-sg" } +} + +# ─── FSx for NetApp ONTAP ───────────────────────────────────────────────────── + +resource "aws_fsx_ontap_file_system" "demo" { + storage_capacity = 1024 + subnet_ids = [var.subnet_id] + preferred_subnet_id = var.subnet_id + deployment_type = "SINGLE_AZ_2" + throughput_capacity = 384 + fsx_admin_password = var.fsxadmin_password + security_group_ids = [aws_security_group.fsx.id] + + tags = { Name = "lakefs-ontap-demo" } +} + +resource "aws_fsx_ontap_storage_virtual_machine" "demo" { + file_system_id = aws_fsx_ontap_file_system.demo.id + name = "fsx" + root_volume_security_style = "UNIX" + svm_admin_password = var.fsxadmin_password + + tags = { Name = "lakefs-ontap-demo-svm" } +} + +resource "aws_fsx_ontap_volume" "demo" { + name = "vol1" + junction_path = "/vol1" + size_in_megabytes = 1024000 + storage_efficiency_enabled = true + storage_virtual_machine_id = aws_fsx_ontap_storage_virtual_machine.demo.id + + tags = { Name = "lakefs-ontap-demo-vol" } +} + +# ─── EC2 Instance ──────────────────────────────────────────────────────────── + +resource "aws_instance" "lakefs" { + ami = data.aws_ami.ubuntu.id + instance_type = "t3.small" + subnet_id = var.subnet_id + associate_public_ip_address = true + vpc_security_group_ids = [aws_security_group.ec2.id] + key_name = var.key_pair_name + + root_block_device { + volume_size = 20 + } + + tags = { Name = "lakefs-ontap-demo" } +} + +# ─── NLB — exposes ONTAP S3 publicly for pre-signed URL support ────────────── + +resource "aws_lb" "ontap_s3" { + name = "lakefs-ontap-s3" + internal = false + load_balancer_type = "network" + subnets = [var.subnet_id] + + tags = { Name = "lakefs-ontap-s3-nlb" } +} + +resource "aws_lb_target_group" "ontap_s3" { + name = "lakefs-ontap-s3-tg" + port = 80 + protocol = "TCP" + vpc_id = var.vpc_id + target_type = "ip" + + health_check { + protocol = "TCP" + port = 80 + } +} + +resource "aws_lb_target_group_attachment" "ontap_s3" { + target_group_arn = aws_lb_target_group.ontap_s3.arn + target_id = tolist(aws_fsx_ontap_storage_virtual_machine.demo.endpoints[0].management[0].ip_addresses)[0] + port = 80 +} + +resource "aws_lb_listener" "ontap_s3" { + load_balancer_arn = aws_lb.ontap_s3.arn + port = 80 + protocol = "TCP" + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.ontap_s3.arn + } +} + +# ─── Elastic IP (so IP never changes between stops/starts) ─────────────────── + +resource "aws_eip" "lakefs" { + instance = aws_instance.lakefs.id + domain = "vpc" + tags = { Name = "lakefs-ontap-demo-eip" } +} diff --git a/01_standalone_examples/netapp-ontap/terraform/outputs.tf b/01_standalone_examples/netapp-ontap/terraform/outputs.tf new file mode 100644 index 000000000..caf7f05b3 --- /dev/null +++ b/01_standalone_examples/netapp-ontap/terraform/outputs.tf @@ -0,0 +1,29 @@ +output "ec2_public_ip" { + description = "EC2 public IP (stable — Elastic IP)" + value = aws_eip.lakefs.public_ip +} + +output "lakefs_ui" { + description = "lakeFS UI URL" + value = "http://${aws_eip.lakefs.public_ip}:8000" +} + +output "fsx_management_ip" { + description = "FSx filesystem management IP (for fsxadmin SSH)" + value = aws_fsx_ontap_file_system.demo.endpoints[0].management[0].ip_addresses +} + +output "svm_management_ip" { + description = "SVM management IP (ONTAP S3 endpoint for lakeFS)" + value = aws_fsx_ontap_storage_virtual_machine.demo.endpoints[0].management[0].ip_addresses +} + +output "ssh_command" { + description = "SSH command to connect to EC2" + value = "ssh -i ~/.ssh/lakefs-ontap-demo.pem ubuntu@${aws_eip.lakefs.public_ip}" +} + +output "ontap_s3_endpoint" { + description = "Public ONTAP S3 endpoint via NLB (use as pre_signed_endpoint in lakeFS config)" + value = "http://${aws_lb.ontap_s3.dns_name}" +} diff --git a/01_standalone_examples/netapp-ontap/terraform/terraform.tfvars.example b/01_standalone_examples/netapp-ontap/terraform/terraform.tfvars.example new file mode 100644 index 000000000..342d9e6ee --- /dev/null +++ b/01_standalone_examples/netapp-ontap/terraform/terraform.tfvars.example @@ -0,0 +1,15 @@ +# Copy this file to terraform.tfvars and fill in your own values. +# terraform.tfvars is gitignored — never commit real secrets or account IDs. + +aws_region = "us-east-1" + +# A VPC and a public subnet in that VPC. +vpc_id = "vpc-xxxxxxxxxxxxxxxxx" +subnet_id = "subnet-xxxxxxxxxxxxxxxxx" + +# An EC2 key pair that already exists in this account/region (for SSH to the lakeFS host). +key_pair_name = "your-ec2-key-pair" + +# Password for the ONTAP fsxadmin / vsadmin users. +# Min 8 chars, must include upper, lower, and a number or special char. +fsxadmin_password = "ChangeMe123!" diff --git a/01_standalone_examples/netapp-ontap/terraform/variables.tf b/01_standalone_examples/netapp-ontap/terraform/variables.tf new file mode 100644 index 000000000..8edad052d --- /dev/null +++ b/01_standalone_examples/netapp-ontap/terraform/variables.tf @@ -0,0 +1,29 @@ +variable "aws_region" { + description = "AWS region to deploy into" + default = "us-east-1" +} + +# ── Account-specific — no defaults; supply via terraform.tfvars ─────────────── + +variable "vpc_id" { + description = "ID of the VPC to deploy into (e.g. vpc-0123456789abcdef0)" + type = string +} + +variable "subnet_id" { + description = "ID of a public subnet in the VPC for the EC2 instance and FSx" + type = string +} + +variable "key_pair_name" { + description = "Name of an EC2 key pair that already exists in this account/region" + type = string +} + +# ── Secrets — no defaults; supply via terraform.tfvars (gitignored) ─────────── + +variable "fsxadmin_password" { + description = "Password for the ONTAP fsxadmin and vsadmin users (min 8 chars, mixed case + number/special)" + type = string + sensitive = true +} From c971ee822e9874949a09e151b4aa34fad77003ef Mon Sep 17 00:00:00 2001 From: Amit Kesarwani <93291915+kesarwam@users.noreply.github.com> Date: Thu, 18 Jun 2026 09:17:54 -0700 Subject: [PATCH 2/5] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f8f2414b9..7425cec90 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,7 @@ Under the [standalone_examples](./01_standalone_examples/) folder are a set of e * [Red Hat OpenShift AI integration](./01_standalone_examples/red-hat-openshift-ai/) * [How to backup, migrate or clone a repo](./01_standalone_examples/backup-migrate-or-clone-repo/) * [Running lakeFS with PostgreSQL as K/V store](./01_standalone_examples/docker-compose-with-postgres/) +* [Running lakeFS with NetApp ONTAP storage](./01_standalone_examples/netapp-ontap/) ## Got Questions or Want to Chat? From d26fc89aeb24fc3b384b18bfabefa59ed3b0f59c Mon Sep 17 00:00:00 2001 From: Amit Kesarwani <93291915+kesarwam@users.noreply.github.com> Date: Thu, 18 Jun 2026 09:20:11 -0700 Subject: [PATCH 3/5] Update 00_index.ipynb --- 00_notebooks/00_index.ipynb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/00_notebooks/00_index.ipynb b/00_notebooks/00_index.ipynb index 966e230a0..f9715003b 100644 --- a/00_notebooks/00_index.ipynb +++ b/00_notebooks/00_index.ipynb @@ -98,7 +98,8 @@ "* [*Kafka* integration](https://github.com/treeverse/lakeFS-samples/blob/main/01_standalone_examples/kafka/)\n", "* [*Flink* integration](https://github.com/treeverse/lakeFS-samples/blob/main/01_standalone_examples/flink/)\n", "* [*Red Hat OpenShift AI* integration](https://github.com/treeverse/lakeFS-samples/blob/main/01_standalone_examples/red-hat-openshift-ai/)\n", - "* [How to **backup, migrate or clone** a repo](https://github.com/treeverse/lakeFS-samples/blob/main/01_standalone_examples/backup-migrate-or-clone-repo/)" + "* [How to **backup, migrate or clone** a repo](https://github.com/treeverse/lakeFS-samples/blob/main/01_standalone_examples/backup-migrate-or-clone-repo/)\n", + "* [Running lakeFS with NetApp ONTAP storage](https://github.com/treeverse/lakeFS-samples/blob/main/01_standalone_examples/netapp-ontap/)" ] }, { From 3edf09ab3efa23a2e789aa6764c01ce836efdd82 Mon Sep 17 00:00:00 2001 From: Amit Kesarwani <93291915+kesarwam@users.noreply.github.com> Date: Thu, 18 Jun 2026 14:10:30 -0700 Subject: [PATCH 4/5] Restructure Step 5 as Enterprise-only, OSS as a footnote. Add everest mount step to demo (PART 2). --- .../netapp-ontap/SETUP_GUIDE.md | 133 ++++++++---------- .../netapp-ontap/terraform/main.tf | 2 + 2 files changed, 59 insertions(+), 76 deletions(-) diff --git a/01_standalone_examples/netapp-ontap/SETUP_GUIDE.md b/01_standalone_examples/netapp-ontap/SETUP_GUIDE.md index 068f899fb..e4e772434 100644 --- a/01_standalone_examples/netapp-ontap/SETUP_GUIDE.md +++ b/01_standalone_examples/netapp-ontap/SETUP_GUIDE.md @@ -81,6 +81,8 @@ This guide sets up a live demo of lakeFS running against a native NetApp ONTAP S - Type: TCP, Port: 80 - Source: `0.0.0.0/0` - Description: `ONTAP S3 HTTP via NLB for pre-signed URLs` + + > ⚠️ **PoC only.** This exposes the ONTAP S3 endpoint to the public internet over plaintext HTTP — pre-signed URL signatures and S3 traffic travel unencrypted. For anything beyond a demo, scope the source to your own IP and front the NLB with TLS. 6. Go to **EC2 → Load Balancers → Create load balancer → Network Load Balancer** - Name: `lakefs-ontap-s3` - Scheme: **Internet-facing** @@ -141,82 +143,13 @@ vserver object-store-server bucket policy statement create -vserver fsx -bucket Type `exit` to leave ONTAP CLI. --- -### Step 5 — Install and Configure lakeFS OSS or lakeFS Enterprise on EC2 - - -### Step 5a — Install and Configure lakeFS OSS on EC2 - -Still on EC2, run: - -```bash -# Install Docker and Python -sudo apt-get update && sudo apt-get install -y docker.io python3-pip -sudo systemctl start docker -sudo usermod -aG docker ubuntu -newgrp docker - -# Start Postgres -docker run -d --name postgres \ - -e POSTGRES_USER=lakefs \ - -e POSTGRES_PASSWORD=lakefs \ - -e POSTGRES_DB=lakefs \ - -p 5432:5432 \ - --restart unless-stopped \ - postgres:15-alpine - -# Install lakeFS -LAKEFS_VERSION=$(curl -s https://api.github.com/repos/treeverse/lakeFS/releases/latest | grep '"tag_name"' | cut -d'"' -f4 | sed 's/v//') -curl -L "https://github.com/treeverse/lakeFS/releases/download/v${LAKEFS_VERSION}/lakeFS_${LAKEFS_VERSION}_Linux_x86_64.tar.gz" | tar xz -sudo mv lakefs /usr/local/bin/ -``` - -Get the SVM management IP from FSx console → Storage Virtual Machines → fsx → Endpoints → Management IP address - -Create the config file (replace the values in CAPS): -```bash -cat > ~/lakefs.yaml << 'EOF' -database: - type: postgres - postgres: - connection_string: "postgresql://lakefs:lakefs@localhost:5432/lakefs?sslmode=disable" - -auth: - encrypt: - secret_key: "a7f3d9e2b8c4f1a6e5d0b3c7f2a9e4d1b6c8f3a2" - -installation: - user_name: admin - access_key_id: "AKIAIOSFODNN7EXAMPLE" - secret_access_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" - -blockstore: - type: s3 - s3: - endpoint: "http://REPLACE_WITH_SVM_MANAGEMENT_IP" - force_path_style: true - region: "us-east-1" - credentials: - access_key_id: "REPLACE_WITH_ONTAP_ACCESS_KEY" - secret_access_key: "REPLACE_WITH_ONTAP_SECRET_KEY" - -logging: - level: "INFO" - format: "text" -EOF -``` - -Start lakeFS: -```bash -nohup lakefs run --config ~/lakefs.yaml > ~/lakefs.log 2>&1 & -``` - ---- +### Step 5 — Install and Configure lakeFS Enterprise on EC2 -### Step 5b — Install and Configure lakeFS Enterprise on EC2 +> This demo uses **lakeFS Enterprise** because the `everest mount` workflow relies on pre-signed URLs resolving from outside the VPC (the reason for the NLB in Step 3), and mount is an Enterprise feature. You'll need a license token. (lakeFS OSS also works against ONTAP for the API-only flow — see the footnote at the end of this step.) -# Copy lakeFS Enterprise license file to EC2: +First, copy the lakeFS Enterprise license file from your machine to EC2: ```bash -scp -i license.token ubuntu@:/home/ubuntu/license.token +scp -i ~/.ssh/lakefs-ontap-demo.pem license.token ubuntu@:/home/ubuntu/ ``` @@ -259,6 +192,8 @@ database: auth: encrypt: secret_key: "a7f3d9e2b8c4f1a6e5d0b3c7f2a9e4d1b6c8f3a2" + ui_config: + rbac: internal installation: user_name: admin @@ -284,9 +219,6 @@ license: path: /home/ubuntu/license.token features: local_rbac: true -auth: - ui_config: - rbac: internal EOF ``` @@ -295,6 +227,14 @@ Start lakeFS: nohup lakefs run --config ~/lakefs.yaml > ~/lakefs.log 2>&1 & ``` +> **Footnote — running lakeFS OSS instead.** If you only need the API-only flow (repos, branches, commits, content-addressed blocks on ONTAP) and don't need `everest mount`, you can run lakeFS OSS instead — no license token, and the NLB / `pre_signed_endpoint` plumbing is unnecessary. Swap the Enterprise install for the OSS binary: +> ```bash +> LAKEFS_VERSION=$(curl -s https://api.github.com/repos/treeverse/lakeFS/releases/latest | grep '"tag_name"' | cut -d'"' -f4 | sed 's/v//') +> curl -L "https://github.com/treeverse/lakeFS/releases/download/v${LAKEFS_VERSION}/lakeFS_${LAKEFS_VERSION}_Linux_x86_64.tar.gz" | tar xz +> sudo mv lakefs /usr/local/bin/ +> ``` +> Then use the same `~/lakefs.yaml` above, minus the `pre_signed_endpoint`, `license`, `features`, and `auth.ui_config` keys. + --- ### Step 6 — Complete lakeFS Setup @@ -334,6 +274,47 @@ python3 demo_flow.py 1. `http://:8000/repositories` — lakeFS UI 2. AWS Console → FSx → `lakefs-ontap-demo` — ONTAP storage view +### Mount the dataset as files (lakeFS Enterprise — `everest mount`): + +This is the payoff for the Enterprise setup: mounting a lakeFS path as a local filesystem. `everest` streams objects on demand via pre-signed URLs — which is why Step 3's NLB and the `pre_signed_endpoint` config are required. + +On EC2, install the `everest` binary (ships with lakeFS Enterprise): +```bash +EVEREST_VERSION=REPLACE_WITH_EVEREST_VERSION_NO +curl -L "https://artifacts.lakefs.io/everest/${EVEREST_VERSION}/everest_${EVEREST_VERSION}_Linux_x86_64.tar.gz" | tar xz +sudo mv everest /usr/local/bin/ +``` + +Point `everest` at lakeFS with the credentials from Step 6: +```bash +cat > ~/.lakectl.yaml << 'EOF' +server: + endpoint_url: http://localhost:8000 +credentials: + access_key_id: + secret_access_key: +EOF +``` + +Mount the `main` branch of the demo repo as local files: +```bash +mkdir -p ~/churn-data +everest mount lakefs://churn-features/main/data ~/churn-data +``` + +Now browse the versioned dataset as if it were on disk — no copy, fetched on demand: +```bash +ls -l ~/churn-data +cat ~/churn-data/customers.csv +``` + +When done, unmount: +```bash +everest umount ~/churn-data +``` + +> If `everest mount` hangs or errors on fetch, the pre-signed URLs aren't reachable — confirm `pre_signed_endpoint` points at the NLB DNS and that the NLB target (the SVM management IP) is healthy. + ### To show raw S3 objects on ONTAP (optional, great for technical audiences): From EC2, install AWS CLI if not already installed: diff --git a/01_standalone_examples/netapp-ontap/terraform/main.tf b/01_standalone_examples/netapp-ontap/terraform/main.tf index a7b644432..309dfc9e3 100644 --- a/01_standalone_examples/netapp-ontap/terraform/main.tf +++ b/01_standalone_examples/netapp-ontap/terraform/main.tf @@ -80,6 +80,8 @@ resource "aws_security_group" "fsx" { description = "All traffic from EC2" } + # PoC only: exposes ONTAP S3 publicly over plaintext HTTP for pre-signed URL + # support. For non-demo use, scope cidr_blocks to a known IP and front with TLS. ingress { from_port = 80 to_port = 80 From f7fae882c48ced06169c164c689e7d9a6992287d Mon Sep 17 00:00:00 2001 From: iddoavn <87393827+iddoavn@users.noreply.github.com> Date: Thu, 18 Jun 2026 17:30:13 -0400 Subject: [PATCH 5/5] Scrub secrets and account-specific data from NetApp ONTAP sample - Remove live ONTAP S3 credentials and real Elastic IP from DEMO_SCRIPT.md - Replace real AWS subnet ID with a placeholder in SETUP_GUIDE.md - Generate lakeFS auth encrypt secret per-run instead of a hardcoded value - Genericize real internal IPs to placeholders - Standardize demo password casing to Netapp1! Co-Authored-By: Claude Opus 4.8 (1M context) --- 01_standalone_examples/netapp-ontap/QUICK_START.md | 4 ++-- 01_standalone_examples/netapp-ontap/SETUP_GUIDE.md | 11 ++++++----- .../netapp-ontap/demo/DEMO_SCRIPT.md | 12 ++++++------ .../netapp-ontap/ontap-setup-notes.md | 6 +++--- .../netapp-ontap/scripts/setup-demo.sh | 5 ++++- 5 files changed, 21 insertions(+), 17 deletions(-) diff --git a/01_standalone_examples/netapp-ontap/QUICK_START.md b/01_standalone_examples/netapp-ontap/QUICK_START.md index bcf4020b3..d4894cd43 100644 --- a/01_standalone_examples/netapp-ontap/QUICK_START.md +++ b/01_standalone_examples/netapp-ontap/QUICK_START.md @@ -96,7 +96,7 @@ Netmask: 255.255.255.0 Gateway: 192.168.64.1 DNS domain: lab.local DNS servers: 8.8.8.8 -Admin password: NetApp1! +Admin password: Netapp1! ``` Wait for `ontap-sim::>` prompt. @@ -104,7 +104,7 @@ Wait for `ontap-sim::>` prompt. **Verify from Mac:** ```bash ping -c 3 192.168.64.11 -ssh admin@192.168.64.11 # password: NetApp1! +ssh admin@192.168.64.11 # password: Netapp1! ``` ## Phase 4 — Configure ONTAP S3 (10 min, inside ONTAP) diff --git a/01_standalone_examples/netapp-ontap/SETUP_GUIDE.md b/01_standalone_examples/netapp-ontap/SETUP_GUIDE.md index e4e772434..984e4bb19 100644 --- a/01_standalone_examples/netapp-ontap/SETUP_GUIDE.md +++ b/01_standalone_examples/netapp-ontap/SETUP_GUIDE.md @@ -31,7 +31,7 @@ This guide sets up a live demo of lakeFS running against a native NetApp ONTAP S - SSD storage: `1024` GiB - Throughput: **Recommended (384 MB/s)** - VPC: `demo-lakefs-vpc` - - Subnet: `demo-lakefs-ontap-public` (`subnet-05960cb8e849935c0`) + - Subnet: `demo-lakefs-ontap-public` (`subnet-xxxxxxxxxxxxxxxxx`) - VPC Security Groups: leave default - File system admin password: `Netapp1!` - SVM name: `fsx` @@ -87,7 +87,7 @@ This guide sets up a live demo of lakeFS running against a native NetApp ONTAP S - Name: `lakefs-ontap-s3` - Scheme: **Internet-facing** - Listener: TCP port 80 - - Target group: IP type, TCP port 80, target = SVM management IP (e.g. `10.20.10.188`) + - Target group: IP type, TCP port 80, target = the SVM management IP (from Step 4) 7. Note the NLB DNS name — you'll need it for the lakeFS config (`pre_signed_endpoint`) > **Terraform users:** Steps 3–7 are handled automatically by `terraform apply`. @@ -99,7 +99,7 @@ This guide sets up a live demo of lakeFS running against a native NetApp ONTAP S Once FSx shows **Available**: 1. Go to **FSx → lakefs-ontap-demo → Administration tab** -2. Note the **Management endpoint IP address** (e.g. `10.0.10.251`) +2. Note the **Management endpoint IP address** — this is your `` 3. SSH into EC2: ```bash @@ -181,7 +181,8 @@ sudo mv lakefs /usr/local/bin/ Get the SVM management IP from FSx console → Storage Virtual Machines → fsx → Endpoints → Management IP address. Get the NLB DNS from `terraform output ontap_s3_endpoint` or the AWS console. -Create the config file (replace the values in CAPS): +Create the config file (replace the values in CAPS). Generate the +`auth.encrypt.secret_key` with `openssl rand -hex 20`: ```bash cat > ~/lakefs.yaml << 'EOF' database: @@ -191,7 +192,7 @@ database: auth: encrypt: - secret_key: "a7f3d9e2b8c4f1a6e5d0b3c7f2a9e4d1b6c8f3a2" + secret_key: "REPLACE_WITH_RANDOM_HEX_SECRET" ui_config: rbac: internal diff --git a/01_standalone_examples/netapp-ontap/demo/DEMO_SCRIPT.md b/01_standalone_examples/netapp-ontap/demo/DEMO_SCRIPT.md index aca022054..1d6357196 100644 --- a/01_standalone_examples/netapp-ontap/demo/DEMO_SCRIPT.md +++ b/01_standalone_examples/netapp-ontap/demo/DEMO_SCRIPT.md @@ -4,7 +4,7 @@ **Duration:** ~10 minutes **Tabs to have open before you start:** 1. AWS Console → FSx → `lakefs-ontap-demo` -2. lakeFS UI → `http://52.87.202.16:8000` +2. lakeFS UI → `http://:8000` 3. Terminal (SSH into EC2, lakeFS running) --- @@ -44,7 +44,7 @@ ## Tab 2 — lakeFS UI (5 min) -**Navigate to `http://52.87.202.16:8000` → churn-features repository.** +**Navigate to `http://:8000` → churn-features repository.** > "Now let's look at lakeFS. This is our repository — think of it like > a GitHub repo, but instead of code files, it contains ML training data. @@ -96,7 +96,7 @@ **Switch to the terminal and SSH into EC2, then SSH to ONTAP:** ``` -ssh fsxadmin@10.0.10.251 +ssh fsxadmin@ ``` Password: `Netapp1!` @@ -136,12 +136,12 @@ exit ``` ```bash -AWS_ACCESS_KEY_ID=WD45C411GSC72OQ8RFPI \ -AWS_SECRET_ACCESS_KEY="HXtcC13CH_qBrq06SBY4Ng5gYGCib6Lq_AN9rN9_" \ +AWS_ACCESS_KEY_ID=$ONTAP_S3_ACCESS_KEY \ +AWS_SECRET_ACCESS_KEY="$ONTAP_S3_SECRET_KEY" \ AWS_DEFAULT_REGION=us-east-1 \ aws s3 ls s3://lakefs-data/ \ --recursive \ - --endpoint-url http://10.0.10.232 + --endpoint-url http:// ``` > "Three levels of visibility in one demo: diff --git a/01_standalone_examples/netapp-ontap/ontap-setup-notes.md b/01_standalone_examples/netapp-ontap/ontap-setup-notes.md index c74bb062f..4a957eb07 100644 --- a/01_standalone_examples/netapp-ontap/ontap-setup-notes.md +++ b/01_standalone_examples/netapp-ontap/ontap-setup-notes.md @@ -165,8 +165,8 @@ Enter DNS server IP addresses []: 8.8.8.8 ... Enter the cluster base license key: [leave blank for simulator — no license needed] ... -Enter new admin password: NetApp1! -Retype new admin password: NetApp1! +Enter new admin password: Netapp1! +Retype new admin password: Netapp1! ``` > **Note on IPs:** Replace `192.168.233.x` with addresses in YOUR hostonly @@ -196,7 +196,7 @@ All commands below are run **inside the ONTAP CLI** (either via SSH or the VM co SSH in: ```bash ssh admin@192.168.233.11 # use your cluster mgmt IP -# password: NetApp1! (or whatever you set) +# password: Netapp1! (or whatever you set) ``` ### 3.1 — Survey the cluster diff --git a/01_standalone_examples/netapp-ontap/scripts/setup-demo.sh b/01_standalone_examples/netapp-ontap/scripts/setup-demo.sh index 5736345d3..06e37299a 100644 --- a/01_standalone_examples/netapp-ontap/scripts/setup-demo.sh +++ b/01_standalone_examples/netapp-ontap/scripts/setup-demo.sh @@ -10,6 +10,9 @@ SVM_IP="${1:?Usage: $0 }" ONTAP_ACCESS_KEY="${2:?missing ONTAP_ACCESS_KEY}" ONTAP_SECRET_KEY="${3:?missing ONTAP_SECRET_KEY}" +# Random encryption secret for lakeFS auth (generated fresh per run). +AUTH_ENCRYPT_SECRET_KEY="$(openssl rand -hex 20)" + echo "==> Installing dependencies..." sudo apt-get update -qq sudo apt-get install -y -qq docker.io python3-pip @@ -41,7 +44,7 @@ database: auth: encrypt: - secret_key: "a7f3d9e2b8c4f1a6e5d0b3c7f2a9e4d1b6c8f3a2" + secret_key: "${AUTH_ENCRYPT_SECRET_KEY}" installation: user_name: admin